fix: use async by default, with occasional def threading (#1015)

* build: add asgiref async_to_sync, plus pytest-asyncio

* build: install script login linger after docker install

* docs: update instructions to automount s3fs

* fix: add user role to user schema, remove appuser logic

* refactor: async initial load read_xlsforms

* refactor: update refs to form --> category in project creation

* refactor: remove redundant projects.utils (replaced)

* refactor: rename add_features --> upload_custom_extract

* fix: update all to async, except BackgroundTasks defs

* fix: update long running central operations to run_in_threadpool

* test: update tests to be async + threadpool

* fix: update expensive generate_task_files to use ThreadPoolExecutor

* fix: use threadpool for generate_task_files

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* refactor: move logic for task schemas into models

* refactor: remove final ref to convert_app_tasks

* refactor: fix loading project summaries without db_obj

* fix: add outline_centroid to task_schemas

* refactor: add task details to project schema output

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/User-Manual-For-Project-Managers.md

@@ -133,7 +133,7 @@ Click on "Next" to proceed.
 ![WhatsApp Image 2023-06-23 at 1 17 37 PM](https://github.com/hotosm/fmtm/assets/97789856/f53d76b4-e6cc-44a4-8c7c-00082eb72693)
 
 14. Select Form . Select the form category you want to use for the field mapping, such as "Data Extract" or any other relevant category.
-    Choose a specific form from the existing forms or upload a custom form if needed.
+    Choose a specific form from the existing categories or upload a custom form if needed.
     Click on "Submit" to proceed.
 
 ![WhatsApp Image 2023-06-23 at 1 37 19 PM](https://github.com/hotosm/fmtm/assets/97789856/f9a4bed7-d1a9-44dd-b2d4-b55f428f9416)

docs/dev/Backend.md

@@ -254,10 +254,12 @@ Access the files like a directory under: `/mnt/fmtm/local`.
 
 To mount permanently, add the following to `/etc/fstab`:
 
-`fmtm-local /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\
-use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\
+`fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\
+use_path_request_style,passwd_file=/home/USERNAME/s3-creds/fmtm-local,\
 url=http://s3.fmtm.localhost:7050 0 0`
 
+> Note: you should replace USERNAME with your linux username.
+
 ## Running JOSM in the dev stack
 
 - Run JOSM with FMTM:

src/backend/app/auth/auth_routes.py

@@ -36,7 +36,7 @@
 
 
 @router.get("/osm_login/")
-def login_url(request: Request, osm_auth=Depends(init_osm_auth)):
+async def login_url(request: Request, osm_auth=Depends(init_osm_auth)):
     """Get Login URL for OSM Oauth Application.
 
     The application must be registered on openstreetmap.org.
@@ -56,7 +56,7 @@ def login_url(request: Request, osm_auth=Depends(init_osm_auth)):
 
 
 @router.get("/callback/")
-def callback(request: Request, osm_auth=Depends(init_osm_auth)):
+async def callback(request: Request, osm_auth=Depends(init_osm_auth)):
     """Performs token exchange between OpenStreetMap and Export tool API.
 
     Core will use Oauth secret key from configuration while deserializing token,
@@ -81,7 +81,7 @@ def callback(request: Request, osm_auth=Depends(init_osm_auth)):
 
 
 @router.get("/me/", response_model=AuthUser)
-def my_data(
+async def my_data(
     db: Session = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
@@ -95,9 +95,11 @@ def my_data(
         user_data(dict): The dict of user data.
     """
     # Save user info in User table
-    user = user_crud.get_user_by_id(db, user_data["id"])
+    user = await user_crud.get_user_by_id(db, user_data["id"])
     if not user:
-        user_by_username = user_crud.get_user_by_username(db, user_data["username"])
+        user_by_username = await user_crud.get_user_by_username(
+            db, user_data["username"]
+        )
         if user_by_username:
             raise HTTPException(
                 status_code=400,
@@ -107,6 +109,7 @@ def my_data(
                 ),
             )
 
+        # Add user to database
         db_user = DbUser(id=user_data["id"], username=user_data["username"])
         db.add(db_user)
         db.commit()

src/backend/app/central/central_crud.py

@@ -53,7 +53,7 @@ def get_odk_project(odk_central: project_schemas.ODKCentral = None):
         log.debug(f"Connecting to ODKCentral: url={url} user={user}")
         project = OdkProject(url, user, pw)
     except Exception as e:
-        log.error(e)
+        log.exception(e)
         raise HTTPException(
             status_code=500, detail=f"Error creating project on ODK Central: {e}"
         ) from e
@@ -144,7 +144,9 @@ def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None
         ) from e
 
 
-def delete_odk_project(project_id: int, odk_central: project_schemas.ODKCentral = None):
+async def delete_odk_project(
+    project_id: int, odk_central: project_schemas.ODKCentral = None
+):
     """Delete a project from a remote ODK Server."""
     # FIXME: when a project is deleted from Central, we have to update the
     # odkid in the projects table
@@ -537,7 +539,9 @@ def generate_updated_xform(
     return outfile
 
 
-def create_qrcode(project_id: int, token: str, name: str, odk_central_url: str = None):
+async def create_qrcode(
+    project_id: int, token: str, name: str, odk_central_url: str = None
+):
     """Create the QR Code for an app-user."""
     if not odk_central_url:
         log.debug("ODKCentral connection variables not set in function")

src/backend/app/central/central_routes.py

@@ -18,6 +18,7 @@
 import json
 
 from fastapi import APIRouter, Depends, HTTPException
+from fastapi.concurrency import run_in_threadpool
 from fastapi.responses import JSONResponse
 from loguru import logger as log
 from sqlalchemy import (
@@ -44,7 +45,8 @@
 async def list_projects():
     """List projects in Central."""
     # TODO update for option to pass credentials by user
-    projects = central_crud.list_odk_projects()
+    # NOTE runs in separate thread using run_in_threadpool
+    projects = await run_in_threadpool(lambda: central_crud.list_odk_projects())
     if projects is None:
         return {"message": "No projects found"}
     return JSONResponse(content={"projects": projects})
@@ -58,8 +60,7 @@ async def create_appuser(
 ):
     """Create an appuser in Central."""
     appuser = central_crud.create_appuser(project_id, name=name)
-    # tasks = tasks_crud.update_qrcode(db, task_id, qrcode['id'])
-    return project_crud.create_qrcode(db, project_id, appuser.get("token"), name)
+    return await project_crud.create_qrcode(db, project_id, appuser.get("token"), name)
 
 
 # @router.get("/list_submissions")
@@ -86,7 +87,8 @@ async def get_form_lists(
     Returns:
     A list of dictionary containing the id and title of each XForm record retrieved from the database.
     """
-    forms = central_crud.get_form_list(db, skip, limit)
+    # NOTE runs in separate thread using run_in_threadpool
+    forms = await run_in_threadpool(lambda: central_crud.get_form_list(db, skip, limit))
     return forms
 
 
@@ -116,6 +118,8 @@ async def download_submissions(
     xforms = central_crud.list_odk_xforms(first.odkid)
     submissions = list()
     for xform in xforms:
+        # FIXME this should be optimised via async or threadpool
+        # FIXME very expensive opteration to run blocking in parallel
         data = central_crud.download_submissions(first.odkid, xform["xmlFormId"])
         # An empty submissions only has the CSV headers
         # headers = data[0]

src/backend/app/db/postgis_utils.py

@@ -15,6 +15,7 @@
 #     You should have received a copy of the GNU General Public License
 #     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
 #
+"""PostGIS and geometry handling helper funcs."""
 
 import datetime
 
@@ -25,11 +26,15 @@
 
 
 def timestamp():
-    """Used in SQL Alchemy models to ensure we refresh timestamp when new models initialised."""
+    """Get the current time.
+
+    Used to insert a current timestamp into Pydantic models.
+    """
     return datetime.datetime.utcnow()
 
 
 def geometry_to_geojson(geometry: Geometry, properties: str = {}, id: int = None):
+    """Convert SQLAlchemy geometry to GeoJSON."""
     if geometry:
         shape = to_shape(geometry)
         geojson = {
@@ -40,15 +45,19 @@ def geometry_to_geojson(geometry: Geometry, properties: str = {}, id: int = None
             # "bbox": shape.bounds,
         }
         return Feature(**geojson)
+    return {}
 
 
-def get_centroid(geometry: Geometry, properties: str = {}):
+def get_centroid(geometry: Geometry, properties: str = {}, id: int = None):
+    """Convert SQLAlchemy geometry to Centroid GeoJSON."""
     if geometry:
         shape = to_shape(geometry)
         point = shape.centroid
         geojson = {
             "type": "Feature",
             "geometry": mapping(point),
             "properties": properties,
+            "id": id,
         }
         return Feature(**geojson)
+    return {}

src/backend/app/main.py

@@ -54,7 +54,7 @@ async def lifespan(app: FastAPI):
     # Startup events
     log.debug("Starting up FastAPI server.")
     log.debug("Reading XLSForms from DB.")
-    read_xlsforms(next(get_db()), xlsforms_path)
+    await read_xlsforms(next(get_db()), xlsforms_path)
 
     yield