diff --git a/docs/User-Manual-For-Project-Managers.md b/docs/User-Manual-For-Project-Managers.md index 30c078ed31..b2b559d3d3 100644 --- a/docs/User-Manual-For-Project-Managers.md +++ b/docs/User-Manual-For-Project-Managers.md @@ -133,7 +133,7 @@ Click on "Next" to proceed. ![WhatsApp Image 2023-06-23 at 1 17 37 PM](https://github.com/hotosm/fmtm/assets/97789856/f53d76b4-e6cc-44a4-8c7c-00082eb72693) 14. Select Form . Select the form category you want to use for the field mapping, such as "Data Extract" or any other relevant category. - Choose a specific form from the existing forms or upload a custom form if needed. + Choose a specific form from the existing categories or upload a custom form if needed. Click on "Submit" to proceed. ![WhatsApp Image 2023-06-23 at 1 37 19 PM](https://github.com/hotosm/fmtm/assets/97789856/f9a4bed7-d1a9-44dd-b2d4-b55f428f9416) diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md index a060fc229e..9fc789166e 100644 --- a/docs/dev/Backend.md +++ b/docs/dev/Backend.md @@ -254,10 +254,12 @@ Access the files like a directory under: `/mnt/fmtm/local`. To mount permanently, add the following to `/etc/fstab`: -`fmtm-local /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\ -use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\ +`fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\ +use_path_request_style,passwd_file=/home/USERNAME/s3-creds/fmtm-local,\ url=http://s3.fmtm.localhost:7050 0 0` +> Note: you should replace USERNAME with your linux username. + ## Running JOSM in the dev stack - Run JOSM with FMTM: diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index fa61715276..32ce5580f4 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -36,7 +36,7 @@ @router.get("/osm_login/") -def login_url(request: Request, osm_auth=Depends(init_osm_auth)): +async def login_url(request: Request, osm_auth=Depends(init_osm_auth)): """Get Login URL for OSM Oauth Application. The application must be registered on openstreetmap.org. @@ -56,7 +56,7 @@ def login_url(request: Request, osm_auth=Depends(init_osm_auth)): @router.get("/callback/") -def callback(request: Request, osm_auth=Depends(init_osm_auth)): +async def callback(request: Request, osm_auth=Depends(init_osm_auth)): """Performs token exchange between OpenStreetMap and Export tool API. Core will use Oauth secret key from configuration while deserializing token, @@ -81,7 +81,7 @@ def callback(request: Request, osm_auth=Depends(init_osm_auth)): @router.get("/me/", response_model=AuthUser) -def my_data( +async def my_data( db: Session = Depends(database.get_db), user_data: AuthUser = Depends(login_required), ): @@ -95,9 +95,11 @@ def my_data( user_data(dict): The dict of user data. """ # Save user info in User table - user = user_crud.get_user_by_id(db, user_data["id"]) + user = await user_crud.get_user_by_id(db, user_data["id"]) if not user: - user_by_username = user_crud.get_user_by_username(db, user_data["username"]) + user_by_username = await user_crud.get_user_by_username( + db, user_data["username"] + ) if user_by_username: raise HTTPException( status_code=400, @@ -107,6 +109,7 @@ def my_data( ), ) + # Add user to database db_user = DbUser(id=user_data["id"], username=user_data["username"]) db.add(db_user) db.commit() diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 6900ac34e8..e599e45e58 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -53,7 +53,7 @@ def get_odk_project(odk_central: project_schemas.ODKCentral = None): log.debug(f"Connecting to ODKCentral: url={url} user={user}") project = OdkProject(url, user, pw) except Exception as e: - log.error(e) + log.exception(e) raise HTTPException( status_code=500, detail=f"Error creating project on ODK Central: {e}" ) from e @@ -144,7 +144,9 @@ def create_odk_project(name: str, odk_central: project_schemas.ODKCentral = None ) from e -def delete_odk_project(project_id: int, odk_central: project_schemas.ODKCentral = None): +async def delete_odk_project( + project_id: int, odk_central: project_schemas.ODKCentral = None +): """Delete a project from a remote ODK Server.""" # FIXME: when a project is deleted from Central, we have to update the # odkid in the projects table @@ -537,7 +539,9 @@ def generate_updated_xform( return outfile -def create_qrcode(project_id: int, token: str, name: str, odk_central_url: str = None): +async def create_qrcode( + project_id: int, token: str, name: str, odk_central_url: str = None +): """Create the QR Code for an app-user.""" if not odk_central_url: log.debug("ODKCentral connection variables not set in function") diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 5bf8c4195d..371c68bf68 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -18,6 +18,7 @@ import json from fastapi import APIRouter, Depends, HTTPException +from fastapi.concurrency import run_in_threadpool from fastapi.responses import JSONResponse from loguru import logger as log from sqlalchemy import ( @@ -44,7 +45,8 @@ async def list_projects(): """List projects in Central.""" # TODO update for option to pass credentials by user - projects = central_crud.list_odk_projects() + # NOTE runs in separate thread using run_in_threadpool + projects = await run_in_threadpool(lambda: central_crud.list_odk_projects()) if projects is None: return {"message": "No projects found"} return JSONResponse(content={"projects": projects}) @@ -58,8 +60,7 @@ async def create_appuser( ): """Create an appuser in Central.""" appuser = central_crud.create_appuser(project_id, name=name) - # tasks = tasks_crud.update_qrcode(db, task_id, qrcode['id']) - return project_crud.create_qrcode(db, project_id, appuser.get("token"), name) + return await project_crud.create_qrcode(db, project_id, appuser.get("token"), name) # @router.get("/list_submissions") @@ -86,7 +87,8 @@ async def get_form_lists( Returns: A list of dictionary containing the id and title of each XForm record retrieved from the database. """ - forms = central_crud.get_form_list(db, skip, limit) + # NOTE runs in separate thread using run_in_threadpool + forms = await run_in_threadpool(lambda: central_crud.get_form_list(db, skip, limit)) return forms @@ -116,6 +118,8 @@ async def download_submissions( xforms = central_crud.list_odk_xforms(first.odkid) submissions = list() for xform in xforms: + # FIXME this should be optimised via async or threadpool + # FIXME very expensive opteration to run blocking in parallel data = central_crud.download_submissions(first.odkid, xform["xmlFormId"]) # An empty submissions only has the CSV headers # headers = data[0] diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index a147a0e765..d1a2b2bc4b 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""PostGIS and geometry handling helper funcs.""" import datetime @@ -25,11 +26,15 @@ def timestamp(): - """Used in SQL Alchemy models to ensure we refresh timestamp when new models initialised.""" + """Get the current time. + + Used to insert a current timestamp into Pydantic models. + """ return datetime.datetime.utcnow() def geometry_to_geojson(geometry: Geometry, properties: str = {}, id: int = None): + """Convert SQLAlchemy geometry to GeoJSON.""" if geometry: shape = to_shape(geometry) geojson = { @@ -40,9 +45,11 @@ def geometry_to_geojson(geometry: Geometry, properties: str = {}, id: int = None # "bbox": shape.bounds, } return Feature(**geojson) + return {} -def get_centroid(geometry: Geometry, properties: str = {}): +def get_centroid(geometry: Geometry, properties: str = {}, id: int = None): + """Convert SQLAlchemy geometry to Centroid GeoJSON.""" if geometry: shape = to_shape(geometry) point = shape.centroid @@ -50,5 +57,7 @@ def get_centroid(geometry: Geometry, properties: str = {}): "type": "Feature", "geometry": mapping(point), "properties": properties, + "id": id, } return Feature(**geojson) + return {} diff --git a/src/backend/app/main.py b/src/backend/app/main.py index a8e93ea0a2..a1cab84f27 100644 --- a/src/backend/app/main.py +++ b/src/backend/app/main.py @@ -54,7 +54,7 @@ async def lifespan(app: FastAPI): # Startup events log.debug("Starting up FastAPI server.") log.debug("Reading XLSForms from DB.") - read_xlsforms(next(get_db()), xlsforms_path) + await read_xlsforms(next(get_db()), xlsforms_path) yield diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index a708a9ad2f..c33864dd15 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -15,12 +15,16 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Logic for FMTM project routes.""" + import io import json import os import time import uuid import zipfile +from asyncio import gather +from concurrent.futures import ThreadPoolExecutor, wait from io import BytesIO from json import dumps, loads from typing import List, Optional @@ -35,6 +39,7 @@ import segno import shapely.wkb as wkblib import sqlalchemy +from asgiref.sync import async_to_sync from fastapi import File, HTTPException, UploadFile from geoalchemy2.shape import from_shape, to_shape from geojson import dump @@ -61,25 +66,25 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import text -from ..central import central_crud -from ..config import settings -from ..db import database, db_models -from ..db.postgis_utils import geometry_to_geojson, timestamp -from ..tasks import tasks_crud -from ..users import user_crud -from . import project_schemas +from app.central import central_crud +from app.config import settings +from app.db import db_models +from app.db.database import get_db +from app.db.postgis_utils import geometry_to_geojson, timestamp +from app.projects import project_schemas +from app.tasks import tasks_crud +from app.users import user_crud QR_CODES_DIR = "QR_codes/" TASK_GEOJSON_DIR = "geojson/" TILESDIR = "/opt/tiles" -def get_projects( +async def get_projects( db: Session, user_id: int, skip: int = 0, limit: int = 100, - db_objects: bool = False, hashtags: List[str] = None, search: str = None, ): @@ -113,12 +118,10 @@ def get_projects( .all() ) project_count = db.query(db_models.DbProject).count() - if db_objects: - return project_count, db_projects - return project_count, convert_to_app_projects(db_projects) + return project_count, await convert_to_app_projects(db_projects) -def get_project_summaries( +async def get_project_summaries( db: Session, user_id: int, skip: int = 0, @@ -126,28 +129,13 @@ def get_project_summaries( hashtags: str = None, search: str = None, ): - # TODO: Just get summaries, something like: - # db_projects = db.query(db_models.DbProject).with_entities( - # db_models.DbProject.id, - # db_models.DbProject.priority, - # db_models.DbProject.total_tasks, - # db_models.DbProject.tasks_mapped, - # db_models.DbProject.tasks_validated, - # db_models.DbProject.tasks_bad_imagery, - # ).join(db_models.DbProject.project_info) \ - # .with_entities( - # db_models.DbProjectInfo.name, - # db_models.DbProjectInfo.short_description) \ - # .filter( - # db_models.DbProject.author_id == user_id).offset(skip).limit(limit).all() - - project_count, db_projects = get_projects( - db, user_id, skip, limit, True, hashtags, search + project_count, db_projects = await get_projects( + db, user_id, skip, limit, hashtags, search ) - return project_count, convert_to_project_summaries(db_projects) + return project_count, await convert_to_project_summaries(db_projects) -def get_project(db: Session, project_id: int): +async def get_project(db: Session, project_id: int): db_project = ( db.query(db_models.DbProject) .filter(db_models.DbProject.id == project_id) @@ -156,27 +144,27 @@ def get_project(db: Session, project_id: int): return db_project -def get_project_by_id(db: Session, project_id: int): +async def get_project_by_id(db: Session, project_id: int): db_project = ( db.query(db_models.DbProject) .filter(db_models.DbProject.id == project_id) .order_by(db_models.DbProject.id) .first() ) - return convert_to_app_project(db_project) + return await convert_to_app_project(db_project) -def get_project_info_by_id(db: Session, project_id: int): +async def get_project_info_by_id(db: Session, project_id: int): db_project_info = ( db.query(db_models.DbProjectInfo) .filter(db_models.DbProjectInfo.project_id == project_id) .order_by(db_models.DbProjectInfo.project_id) .first() ) - return convert_to_app_project_info(db_project_info) + return await convert_to_app_project_info(db_project_info) -def delete_project_by_id(db: Session, project_id: int): +async def delete_project_by_id(db: Session, project_id: int): try: db_project = ( db.query(db_models.DbProject) @@ -193,11 +181,11 @@ def delete_project_by_id(db: Session, project_id: int): return f"Project {project_id} deleted" -def partial_update_project_info( +async def partial_update_project_info( db: Session, project_metadata: project_schemas.ProjectUpdate, project_id ): # Get the project from db - db_project = get_project_by_id(db, project_id) + db_project = await get_project_by_id(db, project_id) # Raise an exception if project is not found. if not db_project: @@ -206,7 +194,7 @@ def partial_update_project_info( ) from None # Get project info - db_project_info = get_project_info_by_id(db, project_id) + db_project_info = await get_project_info_by_id(db, project_id) # Update project informations if project_metadata.name: @@ -220,10 +208,10 @@ def partial_update_project_info( db.commit() db.refresh(db_project) - return convert_to_app_project(db_project) + return await convert_to_app_project(db_project) -def update_project_info( +async def update_project_info( db: Session, project_metadata: project_schemas.ProjectUpload, project_id ): user = project_metadata.author @@ -236,14 +224,14 @@ def update_project_info( raise HTTPException("No project info passed in") # get db user - db_user = user_crud.get_user(db, user.id) + db_user = await user_crud.get_user(db, user.id) if not db_user: raise HTTPException( status_code=400, detail=f"User {user.username} does not exist" ) # verify project exists in db - db_project = get_project_by_id(db, project_id) + db_project = await get_project_by_id(db, project_id) if not db_project: raise HTTPException( status_code=428, detail=f"Project with id {project_id} does not exist" @@ -257,7 +245,7 @@ def update_project_info( db_project.project_name_prefix = project_info.name # get project info - db_project_info = get_project_info_by_id(db, project_id) + db_project_info = await get_project_info_by_id(db, project_id) # Update projects meta informations (name, descriptions) db_project_info.name = project_info.name @@ -267,10 +255,10 @@ def update_project_info( db.commit() db.refresh(db_project) - return convert_to_app_project(db_project) + return await convert_to_app_project(db_project) -def create_project_with_project_info( +async def create_project_with_project_info( db: Session, project_metadata: project_schemas.ProjectUpload, odk_project_id: int ): project_user = project_metadata.author @@ -316,7 +304,7 @@ def create_project_with_project_info( # get db user # TODO: get this from logged in user / request instead, # return 403 (forbidden) if not authorized - db_user = user_crud.get_user(db, project_user.id) + db_user = await user_crud.get_user(db, project_user.id) if not db_user: raise HTTPException( status_code=400, detail=f"User {project_user.username} does not exist" @@ -363,10 +351,10 @@ def create_project_with_project_info( db.commit() db.refresh(db_project) - return convert_to_app_project(db_project) + return await convert_to_app_project(db_project) -def upload_xlsform( +async def upload_xlsform( db: Session, xlsform: str, name: str, @@ -390,10 +378,11 @@ def upload_xlsform( db.commit() return True except Exception as e: + log.error(e) raise HTTPException(status=400, detail={"message": str(e)}) from e -def update_multi_polygon_project_boundary( +async def update_multi_polygon_project_boundary( db: Session, project_id: int, boundary: str, @@ -407,7 +396,7 @@ def update_multi_polygon_project_boundary( boundary = json.loads(boundary) # verify project exists in db - db_project = get_project_by_id(db, project_id) + db_project = await get_project_by_id(db, project_id) if not db_project: log.error(f"Project {project_id} doesn't exist!") return False @@ -466,7 +455,7 @@ def update_multi_polygon_project_boundary( result = db.execute(query) data = result.fetchone() - update_project_location_info(db_project, data[0]) + await update_project_location_info(db_project, data[0]) db.commit() db.refresh(db_project) @@ -478,7 +467,7 @@ def update_multi_polygon_project_boundary( raise HTTPException(e) from e -def preview_tasks(boundary: str, dimension: int): +async def preview_tasks(boundary: str, dimension: int): """Preview tasks by returning a list of task objects.""" """Use a lambda function to remove the "z" dimension from each coordinate in the feature's geometry """ @@ -586,7 +575,7 @@ def remove_z_dimension(coord): return collection -def get_osm_extracts(boundary: str): +async def get_osm_extracts(boundary: str): """Request an extract from raw-data-api and extract the file contents. - The query is posted to raw-data-api and job initiated for fetching the extract. @@ -661,8 +650,8 @@ def get_osm_extracts(boundary: str): return data -def split_into_tasks( - db: Session, outline: str, no_of_buildings: int, has_data_extracts: bool +async def split_into_tasks( + db: Session, project_geojson: str, no_of_buildings: int, has_data_extracts: bool ): """Splits a project into tasks. @@ -675,53 +664,53 @@ def split_into_tasks( Any: A GeoJSON object containing the tasks for the specified project. """ project_id = uuid.uuid4() - all_results = [] - boundary_data = [] - result = [] + boundary_geoms = [] + split_geom_geojson = [] - if outline["type"] == "FeatureCollection": - log.debug("Project boundary GeoJSON = FeatureCollection") - boundary_data.extend(feature["geometry"] for feature in outline["features"]) - result.extend( + async def split_multi_geom_into_tasks(): + # Use asyncio.gather to concurrently process the async generator + split_poly = [ split_polygon_into_tasks( db, project_id, data, no_of_buildings, has_data_extracts ) - for data in boundary_data + for data in boundary_geoms + ] + + # Use asyncio.gather with list to collect results from the async generator + return ( + item for sublist in await gather(*split_poly) for item in sublist if sublist ) - for inner_list in result: - if inner_list: - all_results.extend(iter(inner_list)) + if project_geojson["type"] == "FeatureCollection": + log.debug("Project boundary GeoJSON = FeatureCollection") + boundary_geoms.extend( + feature["geometry"] for feature in project_geojson["features"] + ) + geoms = await split_multi_geom_into_tasks() + split_geom_geojson.extend(geoms) - elif outline["type"] == "GeometryCollection": + elif project_geojson["type"] == "GeometryCollection": log.debug("Project boundary GeoJSON = GeometryCollection") - geometries = outline["geometries"] - boundary_data.extend(iter(geometries)) - result.extend( - split_polygon_into_tasks( - db, project_id, data, no_of_buildings, has_data_extracts - ) - for data in boundary_data - ) - for inner_list in result: - if inner_list: - all_results.extend(iter(inner_list)) + geometries = project_geojson["geometries"] + boundary_geoms.extend(iter(geometries)) + geoms = await split_multi_geom_into_tasks() + split_geom_geojson.extend(geoms) - elif outline["type"] == "Feature": + elif project_geojson["type"] == "Feature": log.debug("Project boundary GeoJSON = Feature") - boundary_data = outline["geometry"] - result = split_polygon_into_tasks( + boundary_geoms = project_geojson["geometry"] + geom = await split_polygon_into_tasks( db, project_id, boundary_data, no_of_buildings, has_data_extracts ) - all_results.extend(iter(result)) + split_geom_geojson.extend(geom) - elif outline["type"] == "Polygon": + elif project_geojson["type"] == "Polygon": log.debug("Project boundary GeoJSON = Polygon") - boundary_data = outline - result = split_polygon_into_tasks( + boundary_geoms = project_geojson + geom = await split_polygon_into_tasks( db, project_id, boundary_data, no_of_buildings, has_data_extracts ) - all_results.extend(result) + split_geom_geojson.extend(geom) else: log.error( @@ -730,11 +719,11 @@ def split_into_tasks( ) return { "type": "FeatureCollection", - "features": all_results, + "features": split_geom_geojson, } -def split_polygon_into_tasks( +async def split_polygon_into_tasks( db: Session, project_id: uuid.UUID, boundary_data: str, @@ -754,7 +743,7 @@ def split_polygon_into_tasks( # TODO update to use flatgeobuf file directly # No need to store in our database if not has_data_extracts: - data = get_osm_extracts(json.dumps(boundary_data)) + data = await get_osm_extracts(json.dumps(boundary_data)) if not data: return None for feature in data["features"]: @@ -807,11 +796,11 @@ def split_polygon_into_tasks( return features -# def update_project_boundary( +# async def update_project_boundary( # db: Session, project_id: int, boundary: str, dimension: int # ): # # verify project exists in db -# db_project = get_project_by_id(db, project_id) +# db_project = await get_project_by_id(db, project_id) # if not db_project: # log.error(f"Project {project_id} doesn't exist!") # return False @@ -879,11 +868,11 @@ def split_polygon_into_tasks( # return True -def update_project_boundary( +async def update_project_boundary( db: Session, project_id: int, boundary: str, dimension: int ): # verify project exists in db - db_project = get_project_by_id(db, project_id) + db_project = await get_project_by_id(db, project_id) if not db_project: log.error(f"Project {project_id} doesn't exist!") return False @@ -931,13 +920,13 @@ def remove_z_dimension(coord): else: outline = shape(features[0]["geometry"]) - update_project_location_info(db_project, outline.wkt) + await update_project_location_info(db_project, outline.wkt) db.commit() db.refresh(db_project) log.debug("Added project boundary!") - result = create_task_grid(db, project_id=project_id, delta=dimension) + result = await create_task_grid(db, project_id=project_id, delta=dimension) # Delete features from the project db.query(db_models.DbFeatures).filter( @@ -971,7 +960,7 @@ def remove_z_dimension(coord): return True -def update_project_with_zip( +async def update_project_with_zip( db: Session, project_id: int, project_name_prefix: str, @@ -1029,7 +1018,7 @@ def update_project_with_zip( ) # verify project exists in db - db_project = get_project_by_id(db, project_id) + db_project = await get_project_by_id(db, project_id) if not db_project: raise HTTPException( status_code=428, detail=f"Project with id {project_id} does not exist" @@ -1040,13 +1029,13 @@ def update_project_with_zip( db_project.task_type_prefix = task_type_prefix # generate outline from file and add to project - outline_shape = get_outline_from_geojson_file_in_zip( + outline_shape = await get_outline_from_geojson_file_in_zip( zip, outline_filename, f"Could not generate Shape from {outline_filename}" ) - update_project_location_info(db_project, outline_shape.wkt) + await update_project_location_info(db_project, outline_shape.wkt) # get all task outlines from file - project_tasks_feature_collection = get_json_from_zip( + project_tasks_feature_collection = await get_json_from_zip( zip, task_outlines_filename, f"Could not generate FeatureCollection from {task_outlines_filename}", @@ -1063,7 +1052,7 @@ def update_project_with_zip( qr_filename = ( f"{project_name_prefix}_{task_type_prefix}__{task_name}.png" ) - db_qr = get_dbqrcode_from_file( + db_qr = await get_dbqrcode_from_file( zip, QR_CODES_DIR + qr_filename, f"QRCode for task {task_name} does not exist. File should be in {qr_filename}", @@ -1071,7 +1060,7 @@ def update_project_with_zip( db.add(db_qr) # save outline - task_outline_shape = get_shape_from_json_str( + task_outline_shape = await get_shape_from_json_str( feature, f"Could not create task outline for {task_name} using {feature}", ) @@ -1080,7 +1069,7 @@ def update_project_with_zip( task_geojson_filename = ( f"{project_name_prefix}_{task_type_prefix}__{task_name}.geojson" ) - task_geojson = get_json_from_zip( + task_geojson = await get_json_from_zip( zip, TASK_GEOJSON_DIR + task_geojson_filename, f"Geojson for task {task_name} does not exist", @@ -1128,7 +1117,7 @@ def update_project_with_zip( # --------------------------- -def read_xlsforms( +async def read_xlsforms( db: Session, directory: str, ): @@ -1171,7 +1160,7 @@ def read_xlsforms( return xlsforms -def get_odk_id_for_project(db: Session, project_id: int): +async def get_odk_id_for_project(db: Session, project_id: int): """Get the odk project id for the fmtm project id.""" project = table( "projects", @@ -1191,7 +1180,7 @@ def get_odk_id_for_project(db: Session, project_id: int): return project_info.odkid -def upload_custom_data_extracts( +async def upload_custom_data_extracts( db: Session, project_id: int, contents: str, @@ -1207,7 +1196,7 @@ def upload_custom_data_extracts( Returns: bool: True if the upload is successful. """ - project = get_project(db, project_id) + project = await get_project(db, project_id) log.debug(f"Uploading custom data extract for project: {project}") if not project: @@ -1281,6 +1270,7 @@ def flatten_dict(d, parent_key="", sep="_"): return items +# NOTE defined as non-async to run in separate thread def generate_task_files( db: Session, project_id: int, @@ -1292,7 +1282,10 @@ def generate_task_files( project_log = log.bind(task="create_project", project_id=project_id) project_log.info(f"Generating files for task {task_id}") - project = get_project(db, project_id) + + get_project_sync = async_to_sync(get_project) + project = get_project_sync(db, project_id) + odk_id = project.odkid project_name = project.project_name_prefix category = project.xform_title @@ -1309,7 +1302,8 @@ def generate_task_files( # prefix should be sent instead of name project_log.info(f"Creating qr code for task {task_id}") - create_qr = create_qrcode( + create_qr_sync = async_to_sync(create_qrcode) + qr_code = create_qr_sync( db, odk_id, appuser.json()["token"], @@ -1317,8 +1311,9 @@ def generate_task_files( odk_credentials.odk_central_url, ) - task = tasks_crud.get_task(db, task_id) - task.qr_code_id = create_qr["qr_code_id"] + get_task_sync = async_to_sync(tasks_crud.get_task) + task = get_task_sync(db, task_id) + task.qr_code_id = qr_code["qr_code_id"] db.commit() db.refresh(task) @@ -1411,7 +1406,7 @@ def generate_task_files( projectId=odk_id, xform=xform_id, actorId=appuser.json()["id"] ) except Exception as e: - log.warning(str(e)) + log.exception(e) project.extract_completed_count += 1 db.commit() @@ -1420,11 +1415,7 @@ def generate_task_files( return True -def generate_task_files_wrapper(project_id, task, xlsform, form_type, odk_credentials): - for db in database.get_db(): - generate_task_files(db, project_id, task, xlsform, form_type, odk_credentials) - - +# NOTE defined as non-async to run in separate thread def generate_appuser_files( db: Session, project_id: int, @@ -1436,6 +1427,7 @@ def generate_appuser_files( background_task_id: Optional[uuid.UUID] = None, ): """Generate the files for each appuser. + QR code, new XForm, and the OSM data extract. Parameters: @@ -1510,7 +1502,8 @@ def generate_appuser_files( # Data Extracts if extracts_contents is not None: project_log.info("Uploading data extracts") - upload_custom_data_extracts(db, project_id, extracts_contents) + upload_extract_sync = async_to_sync(upload_custom_data_extracts) + upload_extract_sync(db, project_id, extracts_contents) else: project = ( @@ -1572,20 +1565,20 @@ def generate_appuser_files( # Generating QR Code, XForm and uploading OSM Extracts to the form. # Creating app users and updating the role of that user. - tasks_list = tasks_crud.get_task_lists(db, project_id) - - # info = get_cpu_info() - # cores = info["count"] - # with concurrent.futures.ThreadPoolExecutor(max_workers=cores) as executor: - # futures = {executor.submit(generate_task_files_wrapper, project_id, task, xlsform, form_type, odk_credentials): task for task in tasks_list} + get_task_lists_sync = async_to_sync(tasks_crud.get_task_lists) + tasks_list = get_task_lists_sync(db, project_id) - # for future in concurrent.futures.as_completed(futures): - # log.debug(f"Waiting for thread to complete..") + # Run with expensive task via threadpool + def wrap_generate_task_files(task): + """Func to wrap and return errors from thread. - for task in tasks_list: + Also passes it's own database session for thread safety. + If we pass a single db session to multiple threads, + there may be inconsistencies or errors. + """ try: generate_task_files( - db, + next(get_db()), project_id, task, xlsform, @@ -1593,30 +1586,40 @@ def generate_appuser_files( odk_credentials, ) except Exception as e: - log.warning(str(e)) - continue + log.exception(str(e)) + + # Use a ThreadPoolExecutor to run the synchronous code in threads + with ThreadPoolExecutor() as executor: + # Submit tasks to the thread pool + futures = [ + executor.submit(wrap_generate_task_files, task) + for task in tasks_list + ] + # Wait for all tasks to complete + wait(futures) - # Update background task status to COMPLETED if background_task_id: - update_background_task_status_in_database( - db, background_task_id, 4 - ) # 4 is COMPLETED + # Update background task status to COMPLETED + update_bg_task_sync = async_to_sync( + update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED except Exception as e: log.warning(str(e)) - # Update background task status to FAILED if background_task_id: - update_background_task_status_in_database( - db, background_task_id, 2, str(e) - ) # 2 is FAILED - + # Update background task status to FAILED + update_bg_task_sync = async_to_sync( + update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED else: # Raise original error if not running in background raise e -def create_qrcode( +async def create_qrcode( db: Session, odk_id: int, token: str, @@ -1625,7 +1628,7 @@ def create_qrcode( ): # Make QR code for an app_user. log.debug(f"Generating base64 encoded QR settings for token: {token}") - qrcode_data = central_crud.create_qrcode( + qrcode_data = await central_crud.create_qrcode( odk_id, token, project_name, odk_central_url ) @@ -1648,7 +1651,7 @@ def create_qrcode( return {"data": qrcode, "id": rows + 1, "qr_code_id": qrdb.id} -def get_project_geometry(db: Session, project_id: int): +async def get_project_geometry(db: Session, project_id: int): """Retrieves the geometry of a project. Args: @@ -1672,7 +1675,7 @@ def get_project_geometry(db: Session, project_id: int): return json.dumps(row) -def get_task_geometry(db: Session, project_id: int): +async def get_task_geometry(db: Session, project_id: int): """Retrieves the geometry of tasks associated with a project. Args: @@ -1682,7 +1685,7 @@ def get_task_geometry(db: Session, project_id: int): Returns: str: A geojson of the task boundaries """ - db_tasks = tasks_crud.get_tasks(db, project_id, None) + db_tasks = await tasks_crud.get_tasks(db, project_id, None) features = [] for task in db_tasks: geom = to_shape(task.outline) @@ -1737,7 +1740,7 @@ async def get_project_features_geojson(db: Session, project_id: int): return features -def create_task_grid(db: Session, project_id: int, delta: int): +async def create_task_grid(db: Session, project_id: int, delta: int): try: # Query DB for project AOI projects = table("projects", column("outline"), column("id")) @@ -1831,10 +1834,10 @@ def create_task_grid(db: Session, project_id: int, delta: int): return out except Exception as e: - log.error(e) + log.exception(e) -def get_json_from_zip(zip, filename: str, error_detail: str): +async def get_json_from_zip(zip, filename: str, error_detail: str): try: with zip.open(filename) as file: data = file.read() @@ -1843,7 +1846,7 @@ def get_json_from_zip(zip, filename: str, error_detail: str): raise HTTPException(status_code=400, detail=f"{error_detail} ----- Error: {e}") -def get_outline_from_geojson_file_in_zip( +async def get_outline_from_geojson_file_in_zip( zip, filename: str, error_detail: str, feature_index: int = 0 ): try: @@ -1864,7 +1867,7 @@ def get_outline_from_geojson_file_in_zip( ) from e -def get_shape_from_json_str(feature: str, error_detail: str): +async def get_shape_from_json_str(feature: str, error_detail: str): try: geom = feature["geometry"] return shape(geom) @@ -1876,7 +1879,7 @@ def get_shape_from_json_str(feature: str, error_detail: str): ) from e -def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str): +async def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str): try: with zip.open(qr_filename) as qr_file: binary_qrcode = qr_file.read() @@ -1903,26 +1906,28 @@ def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str): # TODO: write tests for these -def convert_to_app_project(db_project: db_models.DbProject): - if db_project: - log.debug("Converting db project to app project") - app_project: project_schemas.Project = db_project +async def convert_to_app_project(db_project: db_models.DbProject): + # TODO refactor login to Pydantic models + if not db_project: + log.debug("convert_to_app_project called, but no project provided") + return None - if db_project.outline: - log.debug("Converting project outline to geojson") - app_project.outline_geojson = geometry_to_geojson( - db_project.outline, {"id": db_project.id}, db_project.id - ) + log.debug("Converting db project to app project") + app_project: project_schemas.Project = db_project - app_project.project_tasks = tasks_crud.convert_to_app_tasks(db_project.tasks) + if db_project.outline: + log.debug("Converting project outline to geojson") + app_project.outline_geojson = geometry_to_geojson( + db_project.outline, {"id": db_project.id}, db_project.id + ) - return app_project - else: - log.debug("convert_to_app_project called, but no project provided") - return None + app_project.project_tasks = db_project.tasks + + return app_project -def convert_to_app_project_info(db_project_info: db_models.DbProjectInfo): +async def convert_to_app_project_info(db_project_info: db_models.DbProjectInfo): + # TODO refactor login to Pydantic models if db_project_info: app_project_info: project_schemas.ProjectInfo = db_project_info return app_project_info @@ -1930,19 +1935,25 @@ def convert_to_app_project_info(db_project_info: db_models.DbProjectInfo): return None -def convert_to_app_projects(db_projects: List[db_models.DbProject]): +async def convert_to_app_projects( + db_projects: List[db_models.DbProject], +) -> List[project_schemas.ProjectOut]: + # TODO refactor login to Pydantic models if db_projects and len(db_projects) > 0: - app_projects = [] - for project in db_projects: - if project: - app_projects.append(convert_to_app_project(project)) - app_projects_without_nones = [i for i in app_projects if i is not None] - return app_projects_without_nones + + async def convert_project(project): + return await convert_to_app_project(project) + + app_projects = await gather( + *[convert_project(project) for project in db_projects] + ) + return [project for project in app_projects if project is not None] else: return [] -def convert_to_project_summary(db_project: db_models.DbProject): +async def convert_to_project_summary(db_project: db_models.DbProject): + # TODO refactor login to Pydantic models if db_project: summary: project_schemas.ProjectSummary = db_project @@ -1963,19 +1974,25 @@ def convert_to_project_summary(db_project: db_models.DbProject): return None -def convert_to_project_summaries(db_projects: List[db_models.DbProject]): +async def convert_to_project_summaries( + db_projects: List[db_models.DbProject], +) -> List[project_schemas.ProjectSummary]: + # TODO refactor login to Pydantic models if db_projects and len(db_projects) > 0: - project_summaries = [] - for project in db_projects: - if project: - project_summaries.append(convert_to_project_summary(project)) - app_projects_without_nones = [i for i in project_summaries if i is not None] - return app_projects_without_nones + + async def convert_summary(project): + return await convert_to_project_summary(project) + + project_summaries = await gather( + *[convert_summary(project) for project in db_projects] + ) + return [summary for summary in project_summaries if summary is not None] else: return [] -def convert_to_project_feature(db_project_feature: db_models.DbFeatures): +async def convert_to_project_feature(db_project_feature: db_models.DbFeatures): + # TODO refactor login to Pydantic models if db_project_feature: app_project_feature: project_schemas.Feature = db_project_feature @@ -1991,18 +2008,24 @@ def convert_to_project_feature(db_project_feature: db_models.DbFeatures): return None -def convert_to_project_features(db_project_features: List[db_models.DbFeatures]): +async def convert_to_project_features( + db_project_features: List[db_models.DbFeatures], +) -> List[project_schemas.Feature]: + # TODO refactor login to Pydantic models if db_project_features and len(db_project_features) > 0: - app_project_features = [] - for project_feature in db_project_features: - if project_feature: - app_project_features.append(convert_to_project_feature(project_feature)) - return app_project_features + + async def convert_feature(project_feature): + return await convert_to_project_feature(project_feature) + + app_project_features = await gather( + *[convert_feature(feature) for feature in db_project_features] + ) + return [feature for feature in app_project_features if feature is not None] else: return [] -def get_project_features(db: Session, project_id: int, task_id: int = None): +async def get_project_features(db: Session, project_id: int, task_id: int = None): if task_id: features = ( db.query(db_models.DbFeatures) @@ -2016,7 +2039,7 @@ def get_project_features(db: Session, project_id: int, task_id: int = None): .filter(db_models.DbFeatures.project_id == project_id) .all() ) - return convert_to_project_features(features) + return await convert_to_project_features(features) async def get_extract_completion_count(project_id: int, db: Session): @@ -2066,7 +2089,7 @@ async def insert_background_task_into_database( return task_id -def update_background_task_status_in_database( +async def update_background_task_status_in_database( db: Session, task_id: uuid.UUID, status: int, message: str = None ): """Updates the status of a task in the database @@ -2088,14 +2111,16 @@ def update_background_task_status_in_database( return True -def add_features_into_database( +# NOTE defined as non-async to run in separate thread +def add_custom_extract_to_db( db: Session, features: dict, background_task_id: uuid.UUID, feature_type: str, ): - """Inserts a new task into the database - Params: + """Insert geojson features into db for a project. + + Args: db: database session project_id: id of the project features: features to be added. @@ -2124,9 +2149,10 @@ def add_features_into_database( failure += 1 continue - update_background_task_status_in_database( - db, background_task_id, 4 - ) # 4 is COMPLETED + update_bg_task_sync = async_to_sync( + update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED elif feature_type == "lines": for feature in features["features"]: @@ -2150,24 +2176,23 @@ def add_features_into_database( failure += 1 continue - update_background_task_status_in_database( - db, background_task_id, 4 - ) # 4 is COMPLETED + update_bg_task_sync = async_to_sync( + update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED return True except Exception as e: log.warning(str(e)) - # Update background task status to FAILED - update_background_task_status_in_database( - db, background_task_id, 2, str(e) - ) # 2 is FAILED + update_bg_task_sync = async_to_sync(update_background_task_status_in_database) + update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED async def update_project_form( db: Session, project_id: int, form_type: str, form: UploadFile = File(None) ): - project = get_project(db, project_id) + project = await get_project(db, project_id) category = project.xform_title project_title = project.project_name_prefix odk_id = project.odkid @@ -2254,10 +2279,10 @@ async def update_project_form( db.add(db_feature) db.commit() - tasks_list = tasks_crud.get_task_lists(db, project_id) + tasks_list = await tasks_crud.get_task_lists(db, project_id) for task in tasks_list: - task_obj = tasks_crud.get_task(db, task) + task_obj = await tasks_crud.get_task(db, task) # Get the features for this task. # Postgis query to filter task inside this task outline and of this project @@ -2315,7 +2340,7 @@ async def update_project_form( return True -async def update_odk_credentials( +async def update_odk_credentials_in_db( project_instance: project_schemas.ProjectUpload, odk_central_cred: project_schemas.ODKCentral, odkid: int, @@ -2358,6 +2383,7 @@ async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str) dump(features, jsonfile) +# NOTE defined as non-async to run in separate thread def get_project_tiles( db: Session, project_id: int, @@ -2438,23 +2464,21 @@ def get_project_tiles( db.commit() # Update background task status to COMPLETED - update_background_task_status_in_database( - db, background_task_id, 4 - ) # 4 is COMPLETED + update_bg_task_sync = async_to_sync(update_background_task_status_in_database) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED log.info(f"Tiles generation process completed for project id {project_id}") except Exception as e: - log.error(f"Tiles generation process failed for project id {project_id}") log.error(str(e)) + log.error(f"Tiles generation process failed for project id {project_id}") tile_path_instance.status = 2 db.commit() # Update background task status to FAILED - update_background_task_status_in_database( - db, background_task_id, 2, str(e) - ) # 2 is FAILED + update_bg_task_sync = async_to_sync(update_background_task_status_in_database) + update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED async def get_mbtiles_list(db: Session, project_id: int): @@ -2483,6 +2507,7 @@ async def get_mbtiles_list(db: Session, project_id: int): return processed_tiles_list except Exception as e: + log.error(e) raise HTTPException(status_code=400, detail=str(e)) @@ -2491,7 +2516,7 @@ async def convert_geojson_to_osm(geojson_file: str): return json2osm(geojson_file) -def get_address_from_lat_lon(latitude, longitude): +async def get_address_from_lat_lon(latitude, longitude): """Get address using Nominatim, using lat,lon.""" base_url = "https://nominatim.openstreetmap.org/reverse" @@ -2516,7 +2541,7 @@ def get_address_from_lat_lon(latitude, longitude): return "Address not found." -def update_project_location_info( +async def update_project_location_info( db_project: sqlalchemy.orm.declarative_base, project_boundary: str ): """Update project boundary, centroid, address. @@ -2530,7 +2555,7 @@ def update_project_location_info( db_project.centroid = centroid geometry = wkt.loads(centroid) longitude, latitude = geometry.x, geometry.y - address = get_address_from_lat_lon(latitude, longitude) + address = await get_address_from_lat_lon(latitude, longitude) db_project.location_str = address if address is not None else "" @@ -2602,7 +2627,7 @@ def is_valid_coordinate(coord): raise HTTPException(status_code=400, detail=error_message) -def get_tasks_count(db: Session, project_id: int): +async def get_tasks_count(db: Session, project_id: int): db_task = ( db.query(db_models.DbProject) .filter(db_models.DbProject.id == project_id) @@ -2612,7 +2637,7 @@ def get_tasks_count(db: Session, project_id: int): return task_count -def get_pagintaion(page: int, count: int, results_per_page: int, total: int): +async def get_pagination(page: int, count: int, results_per_page: int, total: int): total_pages = (count + results_per_page - 1) // results_per_page hasNext = (page * results_per_page) < count hasPrev = page > 1 diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index bde0248aa6..8f33360db0 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -15,11 +15,13 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Endpoints for FMTM projects.""" + import json import os import uuid from pathlib import Path -from typing import List, Optional +from typing import Optional from fastapi import ( APIRouter, @@ -46,7 +48,7 @@ from ..db import database, db_models from ..models.enums import TILES_FORMATS, TILES_SOURCE from ..tasks import tasks_crud -from . import project_crud, project_schemas, utils +from . import project_crud, project_schemas from .project_crud import check_crs router = APIRouter( @@ -57,28 +59,30 @@ ) -@router.get("/", response_model=List[project_schemas.ProjectOut]) +@router.get("/", response_model=list[project_schemas.ProjectOut]) async def read_projects( user_id: int = None, skip: int = 0, limit: int = 100, db: Session = Depends(database.get_db), ): - projects = project_crud.get_projects(db, user_id, skip, limit) + project_count, projects = await project_crud.get_projects(db, user_id, skip, limit) return projects -@router.get("/project_details/{project_id}/") +@router.get("/details/{project_id}/") async def get_projet_details(project_id: int, db: Session = Depends(database.get_db)): """Returns the project details. + Also includes ODK project details, so takes extra time to return. + Parameters: project_id: int Returns: Response: Project details. """ - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) if not project: raise HTTPException(status_code=404, details={"Project not found"}) @@ -103,7 +107,7 @@ async def get_projet_details(project_id: int, db: Session = Depends(database.get return { "id": project_id, - "name": odk_details["name"], + "odkName": odk_details["name"], "createdAt": odk_details["createdAt"], "tasks": odk_details["forms"], "lastSubmission": odk_details["lastSubmission"], @@ -111,9 +115,9 @@ async def get_projet_details(project_id: int, db: Session = Depends(database.get } -@router.post("/near_me", response_model=project_schemas.ProjectSummary) -def get_task(lat: float, long: float, user_id: int = None): - return "Coming..." +@router.post("/near_me", response_model=list[project_schemas.ProjectSummary]) +async def get_tasks_near_me(lat: float, long: float, user_id: int = None): + return [project_schemas.ProjectSummary()] @router.get("/summaries", response_model=project_schemas.PaginatedProjectSummaries) @@ -134,11 +138,11 @@ async def read_project_summaries( skip = (page - 1) * results_per_page limit = results_per_page - project_count, projects = project_crud.get_project_summaries( + project_count, projects = await project_crud.get_project_summaries( db, user_id, skip, limit, hashtags, None ) - pagination = project_crud.get_pagintaion( + pagination = await project_crud.get_pagination( page, project_count, results_per_page, total_projects ) project_summaries = [ @@ -173,11 +177,11 @@ async def search_project( skip = (page - 1) * results_per_page limit = results_per_page - project_count, projects = project_crud.get_project_summaries( + project_count, projects = await project_crud.get_project_summaries( db, user_id, skip, limit, hashtags, search ) - pagination = project_crud.get_pagintaion( + pagination = await project_crud.get_pagination( page, project_count, results_per_page, total_projects ) project_summaries = [ @@ -193,12 +197,10 @@ async def search_project( @router.get("/{project_id}", response_model=project_schemas.ProjectOut) async def read_project(project_id: int, db: Session = Depends(database.get_db)): - project = project_crud.get_project_by_id(db, project_id) - if project: - project.project_uuid = uuid.uuid4() - return project - else: + project = await project_crud.get_project_by_id(db, project_id) + if not project: raise HTTPException(status_code=404, detail="Project not found") + return project @router.delete("/delete/{project_id}") @@ -206,7 +208,7 @@ async def delete_project(project_id: int, db: Session = Depends(database.get_db) """Delete a project from ODK Central and the local database.""" # FIXME: should check for error - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -218,9 +220,9 @@ async def delete_project(project_id: int, db: Session = Depends(database.get_db) odk_central_password=project.odk_central_password, ) - central_crud.delete_odk_project(project.odkid, odk_credentials) + await central_crud.delete_odk_project(project.odkid, odk_credentials) - deleted_project = project_crud.delete_project_by_id(db, project_id) + deleted_project = await project_crud.delete_project_by_id(db, project_id) if deleted_project: return deleted_project else: @@ -246,19 +248,17 @@ async def create_project( # TODO check token against user or use token instead of passing user # project_info.project_name_prefix = project_info.project_info.name - project = project_crud.create_project_with_project_info( + project = await project_crud.create_project_with_project_info( db, project_info, odkproject["id"] ) - if project: - project.project_uuid = uuid.uuid4() - return project - else: - raise HTTPException(status_code=404, detail="Project not found") + if not project: + raise HTTPException(status_code=404, detail="Project creation failed") + + return project @router.post("/update_odk_credentials") async def update_odk_credentials( - background_task: BackgroundTasks, odk_central_cred: project_schemas.ODKCentral, project_id: int, db: Session = Depends(database.get_db), @@ -267,39 +267,16 @@ async def update_odk_credentials( if odk_central_cred.odk_central_url.endswith("/"): odk_central_cred.odk_central_url = odk_central_cred.odk_central_url[:-1] - project_instance = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) - if not project_instance: + if not project: raise HTTPException(status_code=404, detail="Project not found") - try: - odkproject = central_crud.create_odk_project( - project_instance.project_info.name, odk_central_cred - ) - log.debug(f"ODKCentral return after update: {odkproject}") - except Exception as e: - log.error(e) - raise HTTPException( - status_code=400, detail="Connection failed to central odk. " - ) from e - - await project_crud.update_odk_credentials( - project_instance, odk_central_cred, odkproject["id"], db + await project_crud.update_odk_credentials_in_db( + project, odk_central_cred, odkproject["id"], db ) - extract_polygon = True if project_instance.data_extract_type == "polygon" else False - project_id = project_instance.id - contents = project_instance.form_xls if project_instance.form_xls else None - - generate_response = await utils.generate_files( - background_tasks=background_task, - project_id=project_id, - extract_polygon=extract_polygon, - upload=contents if contents else None, - db=db, - ) - - return generate_response + return JSONResponse(status_code=200, message={"success": True}) @router.put("/{id}", response_model=project_schemas.ProjectOut) @@ -310,6 +287,9 @@ async def update_project( ): """Update an existing project by ID. + Note: the entire project JSON must be uploaded. + If a partial update is required, use the PATCH method instead. + Parameters: - id: ID of the project to update - author: Author username and id @@ -321,11 +301,10 @@ async def update_project( Raises: - HTTPException with 404 status code if project not found """ - project = project_crud.update_project_info(db, project_info, id) - if project: - return project - else: - raise HTTPException(status_code=404, detail="Project not found") + project = await project_crud.update_project_info(db, project_info, id) + if not project: + raise HTTPException(status_code=422, detail="Project could not be updated") + return project @router.patch("/{id}", response_model=project_schemas.ProjectOut) @@ -349,12 +328,11 @@ async def project_partial_update( - HTTPException with 404 status code if project not found """ # Update project informations - project = project_crud.partial_update_project_info(db, project_info, id) + project = await project_crud.partial_update_project_info(db, project_info, id) - if project: - return project - else: - raise HTTPException(status_code=404, detail="Project not found") + if not project: + raise HTTPException(status_code=422, detail="Project could not be updated") + return project @router.post("/upload_xlsform") @@ -370,7 +348,7 @@ async def upload_custom_xls( """ content = await upload.read() # read file content name = upload.filename.split(".")[0] # get name of file without extension - project_crud.upload_xlsform(db, content, name, category) + await project_crud.upload_xlsform(db, content, name, category) # FIXME: fix return value return {"xform_title": f"{category}"} @@ -404,7 +382,7 @@ async def upload_multi_project_boundary( check_crs(boundary) log.debug("Creating tasks for each polygon in project") - result = project_crud.update_multi_polygon_project_boundary( + result = await project_crud.update_multi_polygon_project_boundary( db, project_id, boundary ) @@ -448,7 +426,7 @@ async def task_split( # Validatiing Coordinate Reference System check_crs(boundary) - result = project_crud.split_into_tasks( + result = await project_crud.split_into_tasks( db, boundary, no_of_buildings, has_data_extracts ) @@ -488,7 +466,9 @@ async def upload_project_boundary( check_crs(boundary) # update project boundary and dimension - result = project_crud.update_project_boundary(db, project_id, boundary, dimension) + result = await project_crud.update_project_boundary( + db, project_id, boundary, dimension + ) if not result: raise HTTPException( status_code=428, detail=f"Project with id {project_id} does not exist" @@ -525,7 +505,9 @@ async def edit_project_boundary( # Validatiing Coordinate Reference System check_crs(boundary) - result = project_crud.update_project_boundary(db, project_id, boundary, dimension) + result = await project_crud.update_project_boundary( + db, project_id, boundary, dimension + ) if not result: raise HTTPException( status_code=428, detail=f"Project with id {project_id} does not exist" @@ -595,7 +577,7 @@ async def generate_files( custom_xls_form = None xform_title = None - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) if not project: raise HTTPException( status_code=428, detail=f"Project with id {project_id} does not exist" @@ -708,6 +690,7 @@ async def get_data_extracts( data_extract = pg.execQuery(boundary) return data_extract except Exception as e: + log.error(e) raise HTTPException(status_code=400, detail=str(e)) @@ -731,8 +714,8 @@ async def update_project_form( return form_updated -@router.get("/{project_id}/features", response_model=List[project_schemas.Feature]) -def get_project_features( +@router.get("/{project_id}/features", response_model=list[project_schemas.Feature]) +async def get_project_features( project_id: int, task_id: int = None, db: Session = Depends(database.get_db), @@ -748,7 +731,7 @@ def get_project_features( Returns: feature(json): JSON object containing a list of features """ - features = project_crud.get_project_features(db, project_id, task_id) + features = await project_crud.get_project_features(db, project_id, task_id) return features @@ -787,7 +770,7 @@ async def generate_log( last_50_logs = filtered_logs[-50:] logs = "\n".join(last_50_logs) - task_count = project_crud.get_tasks_count(db, project_id) + task_count = await project_crud.get_tasks_count(db, project_id) return { "status": task_status.name, "total_tasks": task_count, @@ -845,48 +828,54 @@ async def preview_tasks( # Validatiing Coordinate Reference System check_crs(boundary) - result = project_crud.preview_tasks(boundary, dimension) + result = await project_crud.preview_tasks(boundary, dimension) return result -@router.post("/add_features/") -async def add_features( +@router.post("/upload_custom_extract/") +async def upload_custom_extract( background_tasks: BackgroundTasks, - upload: UploadFile = File(...), + geojson: UploadFile = File(...), feature_type: str = Query( ..., description="Select feature type ", enum=["buildings", "lines"] ), db: Session = Depends(database.get_db), ): - """Add features to a project. + """Upload a custom data extract for a project. + + FIXME + Should this endpoint use upload_custom_data_extract + instead of a new function upload_custom_extract? + Are we duplicating code? + FIXME - This endpoint allows you to add features to a project. + Add osm data extract features to a project. Request Body - 'project_id' (int): the project's id. Required. - - 'upload' (file): Geojson files with the features. Required. + - 'geojson' (file): Geojson files with the features. Required. """ # Validating for .geojson File. - file_name = os.path.splitext(upload.filename) + file_name = os.path.splitext(geojson.filename) file_ext = file_name[1] allowed_extensions = [".geojson", ".json"] if file_ext not in allowed_extensions: raise HTTPException(status_code=400, detail="Provide a valid .geojson file") # read entire file - content = await upload.read() + content = await geojson.read() features = json.loads(content) # Validatiing Coordinate Reference System check_crs(features) # Create task in db and return uuid - log.debug("Creating add_features background task") + log.debug("Creating upload_custom_extract background task") background_task_id = await project_crud.insert_background_task_into_database(db) background_tasks.add_task( - project_crud.add_features_into_database, + project_crud.add_custom_extract_to_db, db, features, background_task_id, @@ -897,7 +886,7 @@ async def add_features( @router.get("/download_form/{project_id}/") async def download_form(project_id: int, db: Session = Depends(database.get_db)): - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -925,7 +914,7 @@ async def update_project_category( ): contents = None - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) if not project: raise HTTPException( status_code=400, detail=f"Project with id {project_id} does not exist" @@ -982,7 +971,7 @@ async def download_project_boundary( Returns: Response: The HTTP response object containing the downloaded file. """ - out = project_crud.get_project_geometry(db, project_id) + out = await project_crud.get_project_geometry(db, project_id) headers = { "Content-Disposition": "attachment; filename=project_outline.geojson", "Content-Type": "application/media", @@ -1004,7 +993,7 @@ async def download_task_boundaries( Returns: Response: The HTTP response object containing the downloaded file. """ - out = project_crud.get_task_geometry(db, project_id) + out = await project_crud.get_task_geometry(db, project_id) headers = { "Content-Disposition": "attachment; filename=project_outline.geojson", @@ -1107,7 +1096,7 @@ async def download_tiles(tile_id: int, db: Session = Depends(database.get_db)): log.info(f"User requested download for tiles: {tiles_path.path}") project_id = tiles_path.project_id - project_name = project_crud.get_project(db, project_id).project_name_prefix + project_name = await project_crud.get_project(db, project_id).project_name_prefix filename = Path(tiles_path.path).name.replace( f"{project_id}_", f"{project_name.replace(' ', '_')}_" ) @@ -1132,7 +1121,7 @@ async def download_task_boundary_osm( Returns: Response: The HTTP response object containing the downloaded file. """ - out = project_crud.get_task_geometry(db, project_id) + out = await project_crud.get_task_geometry(db, project_id) file_path = f"/tmp/{project_id}_task_boundary.geojson" # Write the response content to the file @@ -1161,7 +1150,7 @@ async def project_centroid( project_id (int): The ID of the project. Returns: - List[Tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string. + list[tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string. """ query = text( f"""SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_Centroid(outline)), ST_Y(ST_Centroid(outline))]) AS centroid @@ -1177,12 +1166,15 @@ async def project_centroid( @router.get("/task-status/{uuid}", response_model=project_schemas.BackgroundTaskStatus) async def get_task_status( + task_uuid: str, background_tasks: BackgroundTasks, db: Session = Depends(database.get_db), ): """Get the background task status by passing the task UUID.""" # Get the backgrund task status - task_status, task_message = await project_crud.get_background_task_status(uuid, db) + task_status, task_message = await project_crud.get_background_task_status( + task_uuid, db + ) return project_schemas.BackgroundTaskStatus( status=task_status.name, message=task_message or None, diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 8e23530289..8a81d3aff5 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -137,10 +137,9 @@ class ProjectBase(BaseModel): class ProjectOut(ProjectBase): - project_uuid: uuid.UUID - pass + project_uuid: uuid.UUID = uuid.uuid4() class BackgroundTaskStatus(BaseModel): status: str - message: str + message: Optional[str] = None diff --git a/src/backend/app/projects/utils.py b/src/backend/app/projects/utils.py deleted file mode 100644 index d0db3a566f..0000000000 --- a/src/backend/app/projects/utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import Optional - -from fastapi import ( - BackgroundTasks, - Depends, - File, - Form, - HTTPException, - UploadFile, -) -from sqlalchemy.orm import Session - -from ..db import database -from . import project_crud - - -async def generate_files( - background_tasks: BackgroundTasks, - project_id: int, - extract_polygon: bool = Form(False), - upload: Optional[UploadFile] = File(None), - db: Session = Depends(database.get_db), -): - """Generate required media files tasks in the project based on the provided params. - - Accepts a project ID, category, custom form flag, and an uploaded file as inputs. - The generated files are associated with the project ID and stored in the database. - This function generates qr_code, forms. This utility function also creates an app user for each task and provides the required roles. - Some of the other functionality of this utility includes converting a xls file provided by the user to the xform, - generates osm data extracts and uploads it to the form. - - - Parameters: - - project_id (int): The ID of the project for which files are being generated. This is a required field. - polygon (bool): A boolean flag indicating whether the polygon is extracted or not. - - upload (UploadFile): An uploaded file that is used as input for generating the files. - This is not a required field. A file should be provided if user wants to upload a custom xls form. - - Returns: - Message (str): A success message containing the project ID. - - """ - contents = None - xform_title = None - - project = project_crud.get_project(db, project_id) - if not project: - raise HTTPException( - status_code=428, detail=f"Project with id {project_id} does not exist" - ) - - project.data_extract_type = "polygon" if extract_polygon else "centroid" - db.commit() - - if upload: - file_ext = "xls" - contents = upload - - # Create task in db and return uuid - log.debug(f"Creating generate_files background task for project ID: {project_id}") - background_task_id = await project_crud.insert_background_task_into_database( - db, project_id=project_id - ) - - background_tasks.add_task( - project_crud.generate_appuser_files, - db, - project_id, - extract_polygon, - contents, - None, - xform_title, - file_ext if upload else "xls", - background_task_id, - ) - - return {"Message": f"{project_id}", "task_id": f"{background_task_id}"} diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index ed5b8519ea..c410ad2a50 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -148,7 +148,7 @@ def copy_obj_bucket_to_bucket( ) except Exception as e: - log.error(e) + log.exception(e) log.error(f"Failed to copy object {source_path} to new bucket: {dest_bucket}") return False diff --git a/src/backend/app/submission/submission_crud.py b/src/backend/app/submission/submission_crud.py index 4acf8887ab..b44cb28c1e 100644 --- a/src/backend/app/submission/submission_crud.py +++ b/src/backend/app/submission/submission_crud.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # -import asyncio import concurrent.futures import csv import io @@ -23,8 +22,10 @@ import os import threading import zipfile +from asyncio import gather from pathlib import Path +from asgiref.sync import async_to_sync from fastapi import HTTPException, Response from fastapi.responses import FileResponse from loguru import logger as log @@ -33,7 +34,6 @@ from ..central.central_crud import get_odk_form, get_odk_project from ..projects import project_crud, project_schemas -from ..tasks import tasks_crud def get_submission_of_project(db: Session, project_id: int, task_id: int = None): @@ -41,7 +41,8 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None) This function takes project_id and task_id as a parameter. If task_id is provided, it returns all the submission made to that particular task, else all the submission made in the projects are returned. """ - project_info = project_crud.get_project(db, project_id) + get_project_sync = async_to_sync(project_crud.get_project) + project_info = get_project_sync(db, project_id) # Return empty list if project is not found if not project_info: @@ -98,8 +99,8 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None) return submission_list -def get_forms_of_project(db: Session, project_id: int): - project_info = project_crud.get_project_by_id(db, project_id) +async def get_forms_of_project(db: Session, project_id: int): + project_info = await project_crud.get_project_by_id(db, project_id) # Return empty list if project is not found if not project_info: @@ -112,8 +113,8 @@ def get_forms_of_project(db: Session, project_id: int): return result -def list_app_users_or_project(db: Session, project_id: int): - project_info = project_crud.get_project_by_id(db, project_id) +async def list_app_users_or_project(db: Session, project_id: int): + project_info = await project_crud.get_project_by_id(db, project_id) # Return empty list if project is not found if not project_info: @@ -125,13 +126,6 @@ def list_app_users_or_project(db: Session, project_id: int): return result -def create_zip_file(files, output_file_path): - with zipfile.ZipFile(output_file_path, mode="w") as zip_file: - for file_path in files: - zip_file.write(file_path) - return output_file_path - - # async def convert_json_to_osm_xml(file_path): # jsonin = JsonDump() @@ -205,7 +199,7 @@ async def write_osm_async(features): return osmoutfile data_processing_tasks = [process_entry_async(entry) for entry in data] - processed_features = await asyncio.gather(*data_processing_tasks) + processed_features = await gather(*data_processing_tasks) await write_osm_async(processed_features) return osmoutfile @@ -267,12 +261,14 @@ async def convert_to_osm_for_task(odk_id: int, form_id: int, xform: any): with open(file_path, "wb") as f: f.write(file) - osmoutfile, jsonoutfile = await convert_json_to_osm(file_path) + convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) + osmoutfile, jsonoutfile = convert_json_to_osm_sync(file_path) return osmoutfile, jsonoutfile -async def convert_to_osm(db: Session, project_id: int, task_id: int): - project_info = project_crud.get_project(db, project_id) +def convert_to_osm(db: Session, project_id: int, task_id: int): + get_project_sync = async_to_sync(project_crud.get_project) + project_info = get_project_sync(db, project_id) # Return exception if project is not found if not project_info: @@ -317,7 +313,8 @@ async def convert_to_osm(db: Session, project_id: int, task_id: int): f.write(json.dumps(submission)) # Convert the submission to osm xml format - osmoutfile, jsonoutfile = await convert_json_to_osm(jsoninfile) + convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) + osmoutfile, jsonoutfile = convert_json_to_osm_sync(jsoninfile) if osmoutfile and jsonoutfile: # FIXME: Need to fix this when generating osm file @@ -346,9 +343,10 @@ async def convert_to_osm(db: Session, project_id: int, task_id: int): def download_submission_for_project(db, project_id): - print("Download submission for a project") + log.info(f"Downloading all submissions for a project {project_id}") - project_info = project_crud.get_project(db, project_id) + get_project_sync = async_to_sync(project_crud.get_project) + project_info = get_project_sync(db, project_id) # Return empty list if project is not found if not project_info: @@ -439,7 +437,8 @@ def extract_files(zip_file_path): def get_all_submissions(db: Session, project_id): - project_info = project_crud.get_project(db, project_id) + get_project_sync = async_to_sync(project_crud.get_project) + project_info = get_project_sync(db, project_id) # ODK Credentials odk_credentials = project_schemas.ODKCentral( @@ -450,13 +449,15 @@ def get_all_submissions(db: Session, project_id): project = get_odk_project(odk_credentials) - task_lists = tasks_crud.get_task_lists(db, project_id) + get_task_lists_sync = async_to_sync(get_task_lists) + task_lists = get_task_lists_sync(db, project_id) submissions = project.getAllSubmissions(project_info.odkid, task_lists) return submissions def get_project_submission(db: Session, project_id: int): - project_info = project_crud.get_project(db, project_id) + get_project_sync = async_to_sync(project_crud.get_project) + project_info = get_project_sync(db, project_id) # Return empty list if project is not found if not project_info: @@ -494,8 +495,10 @@ def get_project_submission(db: Session, project_id: int): return submissions -def download_submission(db: Session, project_id: int, task_id: int, export_json: bool): - project_info = project_crud.get_project(db, project_id) +async def download_submission( + db: Session, project_id: int, task_id: int, export_json: bool +): + project_info = await project_crud.get_project(db, project_id) # Return empty list if project is not found if not project_info: @@ -600,13 +603,13 @@ def download_submission(db: Session, project_id: int, task_id: int, export_json: return Response(content=response_content, headers=headers) -def get_submission_points(db: Session, project_id: int, task_id: int = None): +async def get_submission_points(db: Session, project_id: int, task_id: int = None): """Gets the submission points of project. This function takes project_id and task_id as a parameter. If task_id is provided, it returns all the submission points made to that particular task, else all the submission points made in the projects are returned. """ - project_info = project_crud.get_project_by_id(db, project_id) + project_info = await project_crud.get_project_by_id(db, project_id) # Return empty list if project is not found if not project_info: @@ -667,7 +670,7 @@ def get_submission_points(db: Session, project_id: int, task_id: int = None): async def get_submission_count_of_a_project(db: Session, project_id: int): - project_info = project_crud.get_project(db, project_id) + project_info = await project_crud.get_project(db, project_id) # Return empty list if project is not found if not project_info: diff --git a/src/backend/app/submission/submission_routes.py b/src/backend/app/submission/submission_routes.py index c6c14a3036..ed7606ece6 100644 --- a/src/backend/app/submission/submission_routes.py +++ b/src/backend/app/submission/submission_routes.py @@ -19,6 +19,7 @@ import os from fastapi import APIRouter, Depends, Response +from fastapi.concurrency import run_in_threadpool from fastapi.responses import FileResponse from osm_fieldwork.odk_merge import OdkMerge from osm_fieldwork.osmfile import OsmFile @@ -68,7 +69,7 @@ async def list_forms( Returns the list of forms details provided by the central api. """ - return submission_crud.get_forms_of_project(db, project_id) + return await submission_crud.get_forms_of_project(db, project_id) @router.get("/list-app-users") @@ -84,7 +85,7 @@ async def list_app_users( Returns the list of forms details provided by the central api. """ - return submission_crud.list_app_users_or_project(db, project_id) + return await submission_crud.list_app_users_or_project(db, project_id) @router.get("/download") @@ -106,7 +107,9 @@ async def download_submission( file = submission_crud.download_submission_for_project(db, project_id) return FileResponse(file) - return submission_crud.download_submission(db, project_id, task_id, export_json) + return await submission_crud.download_submission( + db, project_id, task_id, export_json + ) @router.get("/submission-points") @@ -121,7 +124,7 @@ async def submission_points( project_id: The ID of the project. This endpoint returns the submission points of this project. task_id: The task_id of the project. This endpoint returns the submission points of this task. """ - return submission_crud.get_submission_points(db, project_id, task_id) + return await submission_crud.get_submission_points(db, project_id, task_id) @router.get("/convert-to-osm") @@ -138,7 +141,11 @@ async def convert_to_osm( If task_id is not provided, this endpoint converts the submission of the whole project. """ - return await submission_crud.convert_to_osm(db, project_id, task_id) + # NOTE runs in separate thread using run_in_threadpool + converted = await run_in_threadpool( + lambda: submission_crud.convert_to_osm(db, project_id, task_id) + ) + return converted @router.get("/get-submission-count/{project_id}") @@ -154,8 +161,11 @@ async def conflate_osm_date( project_id: int, db: Session = Depends(database.get_db), ): - # Submission JSON - submission = submission_crud.get_all_submissions(db, project_id) + # All Submissions JSON + # NOTE runs in separate thread using run_in_threadpool + submission = await run_in_threadpool( + lambda: submission_crud.get_all_submissions(db, project_id) + ) # Data extracta file data_extracts_file = "/tmp/data_extracts_file.geojson" @@ -216,8 +226,11 @@ async def get_osm_xml( if os.path.exists(jsoninfile): os.remove(jsoninfile) - # Submission JSON - submission = submission_crud.get_all_submissions(db, project_id) + # All Submissions JSON + # NOTE runs in separate thread using run_in_threadpool + submission = await run_in_threadpool( + lambda: submission_crud.get_all_submissions(db, project_id) + ) # Write the submission to a file with open(jsoninfile, "w") as f: diff --git a/src/backend/app/tasks/tasks_crud.py b/src/backend/app/tasks/tasks_crud.py index a38f5984e8..ca8c1cb4c1 100644 --- a/src/backend/app/tasks/tasks_crud.py +++ b/src/backend/app/tasks/tasks_crud.py @@ -16,7 +16,6 @@ # along with FMTM. If not, see . # import base64 -from typing import List from fastapi import HTTPException from geoalchemy2.shape import from_shape @@ -24,20 +23,17 @@ from loguru import logger as log from osm_rawdata.postgres import PostgresClient from shapely.geometry import shape -from sqlalchemy import column, select, table from sqlalchemy.orm import Session from sqlalchemy.sql import text from ..central import central_crud from ..db import db_models -from ..db.postgis_utils import geometry_to_geojson from ..models.enums import ( TaskStatus, get_action_for_status_change, verify_valid_status_update, ) from ..projects import project_crud -from ..tasks import tasks_schemas from ..users import user_crud @@ -47,7 +43,7 @@ async def get_task_count_in_project(db: Session, project_id: int): return result.fetchone()[0] -def get_task_lists(db: Session, project_id: int): +async def get_task_lists(db: Session, project_id: int): """Get a list of tasks for a project.""" query = text( """ @@ -65,7 +61,7 @@ def get_task_lists(db: Session, project_id: int): return task_ids -def get_tasks( +async def get_tasks( db: Session, project_id: int, user_id: int, skip: int = 0, limit: int = 1000 ): if project_id: @@ -86,67 +82,81 @@ def get_tasks( ) else: db_tasks = db.query(db_models.DbTask).offset(skip).limit(limit).all() - return convert_to_app_tasks(db_tasks) + return db_tasks -def get_task(db: Session, task_id: int, db_obj: bool = False): - db_task = db.query(db_models.DbTask).filter(db_models.DbTask.id == task_id).first() - if db_obj: - return db_task - return convert_to_app_task(db_task) +async def get_task(db: Session, task_id: int): + return db.query(db_models.DbTask).filter(db_models.DbTask.id == task_id).first() -def update_task_status(db: Session, user_id: int, task_id: int, new_status: TaskStatus): +async def update_task_status( + db: Session, user_id: int, task_id: int, new_status: TaskStatus +): + log.debug(f"Updating task ID {task_id} to status {new_status}") if not user_id: + log.error(f"User id is not present: {user_id}") raise HTTPException(status_code=400, detail="User id required.") - db_user = user_crud.get_user(db, user_id, db_obj=True) + db_user = await user_crud.get_user(db, user_id) if not db_user: - raise HTTPException( - status_code=400, detail=f"User with id {user_id} does not exist." - ) + msg = f"User with id {user_id} does not exist." + log.error(msg) + raise HTTPException(status_code=400, detail=msg) - db_task = get_task(db, task_id, db_obj=True) + db_task = await get_task(db, task_id) + log.debug(f"Returned task from db: {db_task}") if db_task: if db_task.task_status in [ TaskStatus.LOCKED_FOR_MAPPING, TaskStatus.LOCKED_FOR_VALIDATION, ]: + log.debug(f"Task {task_id} currently locked") if not (user_id is not db_task.locked_by): + msg = ( + f"User {user_id} with username {db_user.username} " + "has not locked this task." + ) + log.error(msg) raise HTTPException( status_code=401, - detail=f"User {user_id} with username {db_user.username} has not locked this task.", + detail=msg, ) - if verify_valid_status_update(db_task.task_status, new_status): - # update history prior to updating task - update_history = create_task_history_for_status_change( - db_task, new_status, db_user + if not verify_valid_status_update(db_task.task_status, new_status): + msg = f"{new_status} is not a valid task status" + log.error(msg) + raise HTTPException( + status_code=422, + detail=msg, ) - db.add(update_history) - db_task.task_status = new_status + # update history prior to updating task + update_history = await create_task_history_for_status_change( + db_task, new_status, db_user + ) + db.add(update_history) - if new_status in [ - TaskStatus.LOCKED_FOR_MAPPING, - TaskStatus.LOCKED_FOR_VALIDATION, - ]: - db_task.locked_by = db_user.id - else: - db_task.locked_by = None + db_task.task_status = new_status - if new_status == TaskStatus.MAPPED: - db_task.mapped_by = db_user.id - if new_status == TaskStatus.VALIDATED: - db_task.validated_by = db_user.id - if new_status == TaskStatus.INVALIDATED: - db_task.mapped_by = None + if new_status in [ + TaskStatus.LOCKED_FOR_MAPPING, + TaskStatus.LOCKED_FOR_VALIDATION, + ]: + db_task.locked_by = db_user.id + else: + db_task.locked_by = None - db.commit() - db.refresh(db_task) + if new_status == TaskStatus.MAPPED: + db_task.mapped_by = db_user.id + if new_status == TaskStatus.VALIDATED: + db_task.validated_by = db_user.id + if new_status == TaskStatus.INVALIDATED: + db_task.mapped_by = None - return convert_to_app_task(db_task) + db.commit() + db.refresh(db_task) + return db_task else: raise HTTPException( @@ -160,38 +170,17 @@ def update_task_status(db: Session, user_id: int, task_id: int, new_status: Task # --------------------------- -def update_qrcode( - db: Session, - task_id: int, - qr_id: int, - project_id: int, -): - task = table("tasks", column("qr_code_id"), column("id")) - where = f"task.c.id={task_id}" - value = {"qr_code_id": qr_id} - sql = select( - geoalchemy2.functions.update(task.c.qr_code_id) - .where(text(where)) - .values(text(value)) - ) - log.info(str(sql)) - result = db.execute(sql) - # There should only be one match - if result.rowcount != 1: - log.warning(str(sql)) - return False - - log.info("/tasks/update_qr is partially implemented!") - - -def create_task_history_for_status_change( +async def create_task_history_for_status_change( db_task: db_models.DbTask, new_status: TaskStatus, db_user: db_models.DbUser ): + msg = f"Status changed from {db_task.task_status.name} to {new_status.name} by: {db_user.username}" + log.info(msg) + new_task_history = db_models.DbTaskHistory( project_id=db_task.project_id, task_id=db_task.id, action=get_action_for_status_change(new_status), - action_text=f"Status changed from {db_task.task_status.name} to {new_status.name} by: {db_user.username}", + action_text=msg, actioned_by=db_user, user_id=db_user.id, ) @@ -216,88 +205,11 @@ def create_task_history_for_status_change( # TODO: write tests for these -def convert_to_app_history(db_histories: List[db_models.DbTaskHistory]): - if db_histories: - log.debug("Converting DB Histories to App Histories") - app_histories: List[tasks_schemas.TaskHistoryBase] = [] - for db_history in db_histories: - log.debug( - f"History ID: {db_history.id} | " "Converting DB History to App History" - ) - app_history = db_history - app_history.obj = db_history.action_text - app_histories.append(app_history) - return app_histories - return [] - - -def convert_to_app_task(db_task: db_models.DbTask): - if db_task: - log.debug("") - log.debug( - f"Project ID {db_task.project_id} | Task ID " - f"{db_task.id} | Converting DB Task to App Task" - ) - - app_task: tasks_schemas.Task = db_task - app_task.task_status_str = tasks_schemas.TaskStatusOption[ - app_task.task_status.name - ] - - if db_task.outline: - properties = { - "fid": db_task.project_task_index, - "uid": db_task.id, - "name": db_task.project_task_name, - } - log.debug("Converting task outline to geojson") - app_task.outline_geojson = geometry_to_geojson( - db_task.outline, properties, db_task.id - ) - # app_task.outline_centroid = get_centroid(db_task.outline) - - if db_task.lock_holder: - app_task.locked_by_uid = db_task.lock_holder.id - app_task.locked_by_username = db_task.lock_holder.username - log.debug("Task currently locked by user " f"{app_task.locked_by_username}") - - if db_task.qr_code: - log.debug(f"QR code found for task ID {db_task.id}. Converting to base64") - app_task.qr_code_base64 = base64.b64encode(db_task.qr_code.image) - else: - log.warning(f"No QR code found for task ID {db_task.id}") - app_task.qr_code_base64 = "" - - if db_task.task_history: - app_task.task_history = convert_to_app_history(db_task.task_history) - - return app_task - else: - return None - - -def convert_to_app_tasks(db_tasks: List[db_models.DbTask]): - num_tasks = len(db_tasks) - log.debug(f"Number of tasks in project: {num_tasks}") - - if db_tasks and num_tasks > 0: - log.debug("Converting DB Tasks to App Tasks") - app_tasks = [] - for task in db_tasks: - if task: - app_tasks.append(convert_to_app_task(task)) - app_tasks_without_nones = [i for i in app_tasks if i is not None] - return app_tasks_without_nones - else: - log.debug("No tasks found, skipping DB -> App conversion") - return [] - - -def get_qr_codes_for_task( +async def get_qr_codes_for_task( db: Session, task_id: int, ): - task = get_task(db=db, task_id=task_id) + task = await get_task(db=db, task_id=task_id) if task: if task.qr_code: log.debug(f"QR code found for task ID {task.id}. Converting to base64") @@ -382,9 +294,8 @@ async def update_task_files( async def edit_task_boundary(db: Session, task_id: int, boundary: str): - geometry = boundary["features"][0]["geometry"] - """Update the boundary polyon on the database.""" + geometry = boundary["features"][0]["geometry"] outline = shape(geometry) task = await get_task_by_id(db, task_id) @@ -396,7 +307,7 @@ async def edit_task_boundary(db: Session, task_id: int, boundary: str): # Get category, project_name project_id = task.project_id - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) category = project.xform_title project_name = project.project_name_prefix odk_id = project.odkid diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index e45883b68a..d816cbe298 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -38,20 +38,19 @@ ) -@router.get("/task-list", response_model=List[tasks_schemas.TaskOut]) +@router.get("/task-list", response_model=List[tasks_schemas.Task]) async def read_task_list( project_id: int, limit: int = 1000, db: Session = Depends(database.get_db), ): - tasks = tasks_crud.get_tasks(db, project_id, limit) - if tasks: - return tasks - else: + tasks = await tasks_crud.get_tasks(db, project_id, limit) + if not tasks: raise HTTPException(status_code=404, detail="Tasks not found") + return tasks -@router.get("/", response_model=List[tasks_schemas.TaskOut]) +@router.get("/", response_model=List[tasks_schemas.Task]) async def read_tasks( project_id: int, user_id: int = None, @@ -65,11 +64,10 @@ async def read_tasks( detail="Please provide either user_id OR task_id, not both.", ) - tasks = tasks_crud.get_tasks(db, project_id, user_id, skip, limit) - if tasks: - return tasks - else: + tasks = await tasks_crud.get_tasks(db, project_id, user_id, skip, limit) + if not tasks: raise HTTPException(status_code=404, detail="Tasks not found") + return tasks @router.get("/point_on_surface") @@ -95,38 +93,36 @@ async def get_point_on_surface(project_id: int, db: Session = Depends(database.g return result_dict_list -@router.post("/near_me", response_model=tasks_schemas.TaskOut) -def get_task(lat: float, long: float, project_id: int = None, user_id: int = None): +@router.post("/near_me", response_model=tasks_schemas.Task) +async def get_tasks_near_me( + lat: float, long: float, project_id: int = None, user_id: int = None +): """Get tasks near the requesting user.""" return "Coming..." -@router.get("/{task_id}", response_model=tasks_schemas.TaskOut) +@router.get("/{task_id}", response_model=tasks_schemas.Task) async def read_tasks(task_id: int, db: Session = Depends(database.get_db)): - task = tasks_crud.get_task(db, task_id) - if task: - return task - else: + task = await tasks_crud.get_task(db, task_id) + if not task: raise HTTPException(status_code=404, detail="Task not found") + return task -@router.post("/{task_id}/new_status/{new_status}", response_model=tasks_schemas.TaskOut) +@router.post("/{task_id}/new_status/{new_status}", response_model=tasks_schemas.Task) async def update_task_status( user: user_schemas.User, task_id: int, - new_status: tasks_schemas.TaskStatusOption, + new_status: TaskStatus, db: Session = Depends(database.get_db), ): # TODO verify logged in user user_id = user.id - task = tasks_crud.update_task_status( - db, user_id, task_id, TaskStatus[new_status.name] - ) - if task: - return task - else: + task = await tasks_crud.update_task_status(db, user_id, task_id, new_status) + if not task: raise HTTPException(status_code=404, detail="Task status could not be updated.") + return task @router.post("/task-qr-code/{task_id}") @@ -134,7 +130,7 @@ async def get_qr_code_list( task_id: int, db: Session = Depends(database.get_db), ): - return tasks_crud.get_qr_codes_for_task(db=db, task_id=task_id) + return await tasks_crud.get_qr_codes_for_task(db=db, task_id=task_id) @router.post("/edit-task-boundary") @@ -158,7 +154,7 @@ async def task_features_count( db: Session = Depends(database.get_db), ): # Get the project object. - project = project_crud.get_project(db, project_id) + project = await project_crud.get_project(db, project_id) # ODK Credentials odk_credentials = project_schemas.ODKCentral( diff --git a/src/backend/app/tasks/tasks_schemas.py b/src/backend/app/tasks/tasks_schemas.py index 6a141d7c8f..74548f7de8 100644 --- a/src/backend/app/tasks/tasks_schemas.py +++ b/src/backend/app/tasks/tasks_schemas.py @@ -15,74 +15,121 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Pydantic schemas for FMTM task areas.""" -import enum +import base64 from datetime import datetime -from typing import List +from typing import Any, List, Optional from geojson_pydantic import Feature -from pydantic import BaseModel +from loguru import logger as log +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo +from pydantic.functional_validators import field_validator -from ..models.enums import TaskStatus - - -def get_task_status_strings(): - names = [option.name for option in TaskStatus] - options = {names[i]: names[i] for i in range(len(names))} - return enum.Enum("TaskStatusOptions", options) - - -# Dynamically creates String Enums for API based on Task Status options -TaskStatusOption = get_task_status_strings() +from app.db.postgis_utils import geometry_to_geojson, get_centroid +from app.models.enums import TaskStatus class TaskHistoryBase(BaseModel): + """Task mapping history.""" + id: int action_text: str action_date: datetime class TaskHistoryOut(TaskHistoryBase): + """Task mapping history display.""" + pass -class TaskBasicInfo(BaseModel): - id: int - project_id: int - project_task_index: int - task_status: TaskStatus - locked_by_uid: int = None - locked_by_username: str = None - task_history: List[TaskHistoryBase] +class TaskBase(BaseModel): + """Core fields for a Task.""" + model_config = ConfigDict( + use_enum_values=True, + validate_default=True, + ) + + # Excluded + lock_holder: Any = Field(exclude=True) + outline: Any = Field(exclude=True) + qr_code: Any = Field(exclude=True) -class TaskBase(BaseModel): id: int project_id: int project_task_index: int project_task_name: str - outline_geojson: Feature - # outline_centroid: Feature - # initial_feature_count: int + outline_geojson: Optional[Feature] = None + outline_centroid: Optional[Feature] = None + initial_feature_count: Optional[int] = None task_status: TaskStatus - locked_by_uid: int = None - locked_by_username: str = None - task_history: List[TaskHistoryBase] + locked_by_uid: Optional[int] = None + locked_by_username: Optional[str] = None + task_history: Optional[List[TaskHistoryBase]] = None + + @field_validator("outline_geojson", mode="before") + @classmethod + def get_geojson_from_outline(cls, v: Any, info: ValidationInfo) -> str: + """Get outline_geojson from Shapely geom.""" + if outline := info.data.get("outline"): + properties = { + "fid": info.data.get("project_task_index"), + "uid": info.data.get("id"), + "name": info.data.get("project_task_name"), + } + log.debug("Converting task outline to geojson") + return geometry_to_geojson(outline, properties, info.data.get("id")) + return None + + @field_validator("outline_centroid", mode="before") + @classmethod + def get_centroid_from_outline(cls, v: Any, info: ValidationInfo) -> str: + """Get outline_centroid from Shapely geom.""" + if outline := info.data.get("outline"): + properties = { + "fid": info.data.get("project_task_index"), + "uid": info.data.get("id"), + "name": info.data.get("project_task_name"), + } + log.debug("Converting task outline to geojson") + return get_centroid(outline, properties, info.data.get("id")) + return None + + @field_validator("locked_by_uid", mode="before") + @classmethod + def get_lock_uid(cls, v: int, info: ValidationInfo) -> str: + """Get lock uid from lock_holder details.""" + if lock_holder := info.data.get("lock_holder"): + return lock_holder.get("id") + return None + + @field_validator("locked_by_username", mode="before") + @classmethod + def get_lock_username(cls, v: str, info: ValidationInfo) -> str: + """Get lock username from lock_holder details.""" + if lock_holder := info.data.get("lock_holder"): + return lock_holder.get("username") + return None class Task(TaskBase): - # geometry_geojson: str - qr_code_base64: str - task_status_str: TaskStatusOption - pass - - -class TaskOut(TaskBase): - qr_code_base64: str - task_status_str: TaskStatusOption - pass - - -class TaskDetails(TaskBase): - pass + """Task details plus base64 QR codes.""" + + qr_code_base64: Optional[str] = None + + @field_validator("qr_code_base64", mode="before") + @classmethod + def get_qrcode_base64(cls, v: Any, info: ValidationInfo) -> str: + """Get base64 encoded qrcode.""" + if qr_code := info.data.get("qr_code"): + log.debug( + f"QR code found for task ID {info.data.get('id')}. " + "Converting to base64" + ) + return base64.b64encode(qr_code.image) + else: + log.warning(f"No QR code found for task ID {info.data.get('id')}") + return "" diff --git a/src/backend/app/users/user_crud.py b/src/backend/app/users/user_crud.py index 366ac8a2c8..20d5a420b3 100644 --- a/src/backend/app/users/user_crud.py +++ b/src/backend/app/users/user_crud.py @@ -15,7 +15,8 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # -from typing import List +"""Logic for user routes.""" + from sqlalchemy.orm import Session @@ -27,52 +28,25 @@ # -------------- -def get_users(db: Session, skip: int = 0, limit: int = 100): - db_users = db.query(db_models.DbUser).offset(skip).limit(limit).all() - return convert_to_app_user(db_users) if db_users else [] +async def get_users(db: Session, skip: int = 0, limit: int = 100): + """Get all users.""" + return db.query(db_models.DbUser).offset(skip).limit(limit).all() -def get_user(db: Session, user_id: int, db_obj: bool = False): - db_user = db.query(db_models.DbUser).filter(db_models.DbUser.id == user_id).first() - if db_obj: - return db_user - return convert_to_app_user(db_user) +async def get_user(db: Session, user_id: int): + """Get a single user by user id.""" + return db.query(db_models.DbUser).filter(db_models.DbUser.id == user_id).first() -def get_user_by_username(db: Session, username: str): - db_user = ( +async def get_user_by_username(db: Session, username: str): + """Get a single user by username.""" + return ( db.query(db_models.DbUser).filter(db_models.DbUser.username == username).first() ) - return convert_to_app_user(db_user) - - -# -------------------- -# ---- CONVERTERS ---- -# -------------------- - - -# TODO: write tests for these -def convert_to_app_user(db_user: db_models.DbUser): - if db_user: - app_user: user_schemas.User = db_user - return app_user - else: - return None -def convert_to_app_users(db_users: List[db_models.DbUser]): - if db_users and len(db_users) > 0: - app_users = [] - for user in db_users: - if user: - app_users.append(convert_to_app_user(user)) - app_users_without_nones = [i for i in app_users if i is not None] - return app_users_without_nones - else: - return [] - - -def get_user_role_by_user_id(db: Session, user_id: int): +async def get_user_role_by_user_id(db: Session, user_id: int): + """Return the user role for a given user ID.""" db_user_role = ( db.query(db_models.DbUserRoles) .filter(db_models.DbUserRoles.user_id == user_id) @@ -84,6 +58,7 @@ def get_user_role_by_user_id(db: Session, user_id: int): async def create_user_roles(user_role: user_schemas.UserRoles, db: Session): + """Assign a user a role.""" db_user_role = db_models.DbUserRoles( user_id=user_role.user_id, role=user_role.role, @@ -95,8 +70,3 @@ async def create_user_roles(user_role: user_schemas.UserRoles, db: Session): db.commit() db.refresh(db_user_role) return db_user_role - - -def get_user_by_id(db: Session, user_id: int): - db_user = db.query(db_models.DbUser).filter(db_models.DbUser.id == user_id).first() - return db_user diff --git a/src/backend/app/users/user_routes.py b/src/backend/app/users/user_routes.py index dbecc8a778..3b6f0d4d15 100644 --- a/src/backend/app/users/user_routes.py +++ b/src/backend/app/users/user_routes.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Endpoints for users and role.""" from typing import List @@ -34,61 +35,93 @@ @router.get("/", response_model=List[user_schemas.UserOut]) -def get_users( - username: str = "", +async def get_users( skip: int = 0, limit: int = 100, db: Session = Depends(database.get_db), ): - users = user_crud.get_users(db, skip=skip, limit=limit) + """Get all user details.""" + users = await user_crud.get_users(db, skip=skip, limit=limit) + + if not users: + raise HTTPException(status_code=404, detail="No users found") + return users - # TODO error thrown when no users are in db @router.get("/{id}", response_model=user_schemas.UserOut) -async def get_user_by_id(id: int, db: Session = Depends(database.get_db)): - user = user_crud.get_user(db, user_id=id) - if user: - user.role = user.role.name - return user - else: - raise HTTPException(status_code=404, detail="User not found") +async def get_user_by_identifier(id: str, db: Session = Depends(database.get_db)): + """Get a single users details. + + The OSM ID should be used. + If this is not known, the endpoint falls back to searching + for the username. + """ + user = None + # Search by ID + try: + osm_id = int(id) + user = await user_crud.get_user(db, user_id=osm_id) + except ValueError: + # Skip if not a valid integer + pass + + if not user: + # Search by Username + user = await user_crud.get_user_by_username(db, username=id) + + # No user found + if not user: + raise HTTPException(status_code=404, detail="User not found") + + return user @router.post("/user-role") async def create_user_role( user_role: user_schemas.UserRoles, db: Session = Depends(database.get_db) ): - """This api creates a new role for the user. - The role can be Admin, Organization Admin, Field Admin, Mapper, Validator or Read Only. + """Create a new user role. + + # FIXME is this endpoint really necessary? - Request Parameters: + The role can be: + - Admin + - Organization Admin + - Field Admin + - Mapper + - Validator + - Read Only + + The request param `user_role` is a json of user_id, organization_id, + project_id, user_role: user_id (required): ID of the user for whom the role is being created - organization_id (optional): ID of the organization for which the user is being assigned a role - project_id (optional): ID of the project for which the user is being assigned a role - role (required): Role being assigned to the user + organization_id (optional): ID of the organization for which the + user is being assigned a role + project_id (optional): ID of the project for which the user is + being assigned a role + user_role (required): Role being assigned to the user Response: Status Code 200 (OK): If the role is successfully created Status Code 400 (Bad Request): If the user is already assigned a role """ - existing_user_role = user_crud.get_user_role_by_user_id( + existing_user_role = await user_crud.get_user_role_by_user_id( db, user_id=user_role.user_id ) if existing_user_role is not None: raise HTTPException(status_code=400, detail="User is already assigned a role") - user = user_crud.get_user(db, user_id=user_role.user_id) + user = await user_crud.get_user(db, user_id=user_role.user_id) if user is None: raise HTTPException(status_code=404, detail="User not found") - print("Hellooo") - return await user_crud.create_user_roles(user_role, db) @router.get("/user-role-options/") async def get_user_roles(): + """Check for available user role options.""" user_roles = {} for role in UserRoleEnum: user_roles[role.name] = role.value diff --git a/src/backend/app/users/user_schemas.py b/src/backend/app/users/user_schemas.py index 7ee1c92b7a..df16a42c04 100644 --- a/src/backend/app/users/user_schemas.py +++ b/src/backend/app/users/user_schemas.py @@ -15,31 +15,44 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Pydantic models for Users and Roles.""" from typing import Optional from pydantic import BaseModel +from app.models.enums import UserRole + class UserBase(BaseModel): + """Username only.""" + username: str class User(UserBase): + """User with ID.""" + id: int class UserOut(UserBase): + """User with ID and role.""" + id: int - role: str + role: UserRole class UserRole(BaseModel): - role: str + """User role only.""" + + role: UserRole class UserRoles(BaseModel): + """User details with role, org, and associated project.""" + user_id: int organization_id: Optional[int] = None project_id: Optional[int] = None - role: str + role: UserRole diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 959f911a3d..4ec0a2731a 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -6,7 +6,7 @@ groups = ["default", "debug", "dev", "docs", "test"] cross_platform = true static_urls = false lock_version = "4.3" -content_hash = "sha256:bd5a15380dcce6d93f53afd8689cce70f4c04616a367d34ac7370e2caa830b2d" +content_hash = "sha256:23547250f87f2c18c99b5f09aded65d3591631198321c15c91cf966ce41a5b62" [[package]] name = "annotated-types" @@ -52,6 +52,19 @@ files = [ {file = "argcomplete-3.1.4.tar.gz", hash = "sha256:72558ba729e4c468572609817226fb0a6e7e9a0a7d477b882be168c0b4a62b94"}, ] +[[package]] +name = "asgiref" +version = "3.7.2" +requires_python = ">=3.7" +summary = "ASGI specs, helper code, and adapters" +dependencies = [ + "typing-extensions>=4; python_version < \"3.11\"", +] +files = [ + {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"}, + {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"}, +] + [[package]] name = "asttokens" version = "2.4.1" @@ -1449,6 +1462,19 @@ files = [ {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] +[[package]] +name = "pytest-asyncio" +version = "0.21.1" +requires_python = ">=3.7" +summary = "Pytest support for asyncio" +dependencies = [ + "pytest>=7.0.0", +] +files = [ + {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"}, + {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"}, +] + [[package]] name = "python-dateutil" version = "2.8.2" diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index a9836c552c..322fcb68fc 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "osm-rawdata==0.1.5", "minio>=7.1.17", "pyproj>=3.6.1", + "asgiref>=3.7.2", ] requires-python = ">=3.10,<3.12" readme = "../../README.md" @@ -69,6 +70,7 @@ dev = [ test = [ "pytest>=7.2.2", "httpx>=0.23.3", + "pytest-asyncio>=0.21.1", ] debug = [ "ipdb>=0.13.13", @@ -113,6 +115,7 @@ addopts = "-ra -q" testpaths = [ "tests", ] +asyncio_mode="auto" [tool.commitizen] name = "cz_conventional_commits" diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index 180de4f959..d68674ca98 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # +"""Configuration and fixtures for PyTest.""" import logging import os @@ -32,7 +33,7 @@ from app.config import settings from app.db.database import Base, get_db from app.db.db_models import DbOrganisation, DbUser -from app.main import api, get_application +from app.main import get_application from app.projects import project_crud from app.projects.project_schemas import ODKCentral, ProjectInfo, ProjectUpload from app.users.user_schemas import User @@ -51,12 +52,13 @@ def pytest_configure(config): @pytest.fixture(autouse=True) def app() -> Generator[FastAPI, Any, None]: - """Create a fresh database on each test case.""" + """Get the FastAPI test server.""" yield get_application() @pytest.fixture(scope="session") def db_engine(): + """The SQLAlchemy database engine to init.""" engine = create_engine(settings.FMTM_DB_URL.unicode_string()) if not database_exists: create_database(engine.url) @@ -67,6 +69,7 @@ def db_engine(): @pytest.fixture(scope="function") def db(db_engine): + """Database session using db_engine.""" connection = db_engine.connect() # begin a non-ORM transaction @@ -83,6 +86,7 @@ def db(db_engine): @pytest.fixture(scope="function") def user(db): + """A test user.""" db_user = DbUser(id=100, username="test_user") db.add(db_user) db.commit() @@ -91,6 +95,7 @@ def user(db): @pytest.fixture(scope="function") def organization(db): + """A test organisation.""" db_org = DbOrganisation( name="test_org_qwerty", slug="test_qwerty", @@ -104,7 +109,8 @@ def organization(db): @pytest.fixture(scope="function") -def project(db, user, organization): +async def project(db, user, organization): + """A test project, using the test user and org.""" project_metadata = ProjectUpload( author=User(username=user.username, id=user.id), project_info=ProjectInfo( @@ -136,17 +142,18 @@ def project(db, user, organization): log.debug(f"ODK project returned: {odkproject}") assert odkproject is not None except Exception as e: - log.error(e) + log.exception(e) pytest.fail(f"Test failed with exception: {str(e)}") # Create FMTM Project try: - new_project = project_crud.create_project_with_project_info( + new_project = await project_crud.create_project_with_project_info( db, project_metadata, odkproject["id"] ) log.debug(f"Project returned: {new_project.__dict__}") assert new_project is not None except Exception as e: + log.exception(e) pytest.fail(f"Test failed with exception: {str(e)}") return new_project @@ -174,8 +181,9 @@ def project(db, user, organization): @pytest.fixture(scope="function") -def client(db): - api.dependency_overrides[get_db] = lambda: db +def client(app, db): + """The FastAPI test server.""" + app.dependency_overrides[get_db] = lambda: db - with TestClient(api) as c: + with TestClient(app) as c: yield c diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index 30a6c95e69..dbd5a0b263 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -1,11 +1,30 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# +"""Tests for project routes.""" + import json import os import uuid import zipfile +from concurrent.futures import ThreadPoolExecutor, wait from unittest.mock import Mock, patch import pytest -from fastapi import FastAPI from geoalchemy2.elements import WKBElement from loguru import logger as log from shapely import Polygon, wkb @@ -21,21 +40,9 @@ odk_central_user = os.getenv("ODK_CENTRAL_USER") odk_central_password = os.getenv("ODK_CENTRAL_PASSWD") -app = FastAPI() - - -class MockSession: - def add(self, item): - pass - - def commit(self): - pass - - def refresh(self, item): - pass - -def test_create_project(client, organization, user): +async def test_create_project(client, organization, user): + """Test project creation endpoint.""" project_data = { "author": {"username": user.username, "id": user.id}, "project_info": { @@ -61,7 +68,8 @@ def test_create_project(client, organization, user): assert "id" in response_data -def test_create_odk_project(): +async def test_create_odk_project(): + """Test creating an odk central project.""" mock_project = Mock() mock_project.createProject.return_value = {"status": "success"} @@ -72,7 +80,8 @@ def test_create_odk_project(): mock_project.createProject.assert_called_once_with("Test Project") -def test_convert_to_app_project(): +async def test_convert_to_app_project(): + """Test conversion ot app project.""" polygon = Polygon( [ (85.924707758, 26.727727503), @@ -91,7 +100,7 @@ def test_convert_to_app_project(): outline=wkb_outline, ) - result = project_crud.convert_to_app_project(mock_db_project) + result = await project_crud.convert_to_app_project(mock_db_project) assert result is not None assert isinstance(result, db_models.DbProject) @@ -102,12 +111,14 @@ def test_convert_to_app_project(): assert isinstance(result.project_tasks, list) -def test_create_project_with_project_info(db, project): +async def test_create_project_with_project_info(db, project): + """Test creating a project with all project info.""" assert isinstance(project.id, int) assert isinstance(project.project_name_prefix, str) -def test_generate_appuser_files(db, project): +async def test_generate_appuser_files(db, project): + """Test generate all appuser files (during creation).""" odk_credentials = { "odk_central_url": odk_central_url, "odk_central_user": odk_central_user, @@ -136,12 +147,12 @@ def test_generate_appuser_files(db, project): ) ) log.debug(f"Creating project boundary: {boundary_geojson}") - boundary_created = project_crud.update_project_boundary( + boundary_created = await project_crud.update_project_boundary( db, project_id, boundary_geojson, 500 ) assert boundary_created is True # Check updated locations - db_project = project_crud.get_project_by_id(db, project_id) + db_project = await project_crud.get_project_by_id(db, project_id) # Outline project_outline = db_project.outline.data.tobytes() file_outline = shape(boundary_geojson) @@ -160,24 +171,40 @@ def test_generate_appuser_files(db, project): # Upload data extracts log.debug(f"Uploading custom data extracts: {str(data_extracts)[:100]}...") - data_extract_uploaded = project_crud.upload_custom_data_extracts( + data_extract_uploaded = await project_crud.upload_custom_data_extracts( db, project_id, data_extracts ) assert data_extract_uploaded is True # Get project tasks list - task_list = tasks_crud.get_task_lists(db, project_id) + task_list = await tasks_crud.get_task_lists(db, project_id) assert isinstance(task_list, list) # Provide custom xlsform file path xlsform_file = f"{test_data_path}/buildings.xls" - # Generate project task files - for task in task_list: - task_list = project_crud.generate_task_files( - db, project_id, task, xlsform_file, "xls", odk_credentials - ) - assert task_list is True + # Generate project task files using threadpool + with ThreadPoolExecutor() as executor: + # Submit tasks to the thread pool + futures = [ + executor.submit( + project_crud.generate_task_files, + db, + project_id, + task, + xlsform_file, + "xls", + odk_credentials, + ) + for task in task_list + ] + + # Wait for all tasks to complete + wait(futures) + + # Check the results, assuming generate_task_files returns a boolean + results = [future.result() for future in futures] + assert all(results) # Generate appuser files test_data = { @@ -190,9 +217,14 @@ def test_generate_appuser_files(db, project): "form_type": "example_form_type", "background_task_id": uuid.uuid4(), } - result = project_crud.generate_appuser_files(**test_data) + # Generate appuser using threadpool + with ThreadPoolExecutor() as executor: + future = executor.submit(project_crud.generate_appuser_files, **test_data) + result = future.result() + assert result is None if __name__ == "__main__": + """Main func if file invoked directly.""" pytest.main() diff --git a/src/frontend/public/install.sh b/src/frontend/public/install.sh index 0532fe2c01..671383de11 100644 --- a/src/frontend/public/install.sh +++ b/src/frontend/public/install.sh @@ -114,10 +114,6 @@ check_user_not_root() { useradd -m -d /home/svcfmtm -s /bin/bash svcfmtm 2>/dev/null fi - echo - yellow_echo "Enable login linger for user svcfmtm (docker if logged out)." - loginctl enable-linger svcfmtm - echo yellow_echo "Temporarily adding to sudoers list." echo "svcfmtm ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/fmtm-sudoers >/dev/null @@ -394,6 +390,10 @@ install_docker() { restart_docker_rootless update_docker_ps_format add_vars_to_bashrc + # Enable docker daemon to remain after ssh disconnect + echo + yellow_echo "Enable login linger for user $(whoami) (docker daemon on ssh disconnect)." + loginctl enable-linger "$(whoami)" else heading_echo "Docker is Required. Aborting." "red" exit 1 diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index 5b8a24c485..9f46b7a725 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -53,7 +53,9 @@ const CreateProjectService: Function = ( const dataExtractFormData = new FormData(); dataExtractFormData.append('upload', dataExtractFile); const postDataExtract = await axios.post( - `${import.meta.env.VITE_API_URL}/projects/add_features/?project_id=${resp.id}&feature_type=buildings`, + `${import.meta.env.VITE_API_URL}/projects/upload_custom_extract/?project_id=${ + resp.id + }&feature_type=buildings`, dataExtractFormData, ); } @@ -61,7 +63,7 @@ const CreateProjectService: Function = ( const lineExtractFormData = new FormData(); lineExtractFormData.append('upload', lineExtractFile); const postLineExtract = await axios.post( - `${import.meta.env.VITE_API_URL}/projects/add_features/?project_id=${resp.id}&feature_type=lines`, + `${import.meta.env.VITE_API_URL}/projects/upload_custom_extract/?project_id=${resp.id}&feature_type=lines`, lineExtractFormData, ); } diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 1d7a04455f..4fdc543aed 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -19,7 +19,7 @@ export const ProjectById = (existingProjectList, projectId) => { return { id: data.id, project_task_name: data.project_task_name, - task_status_str: data.task_status_str, + task_status: data.task_status, outline_geojson: data.outline_geojson, outline_centroid: data.outline_centroid, task_history: data.task_history, diff --git a/src/frontend/src/api/ProjectTaskStatus.js b/src/frontend/src/api/ProjectTaskStatus.js index 27ae9a5a3e..c4821ba357 100755 --- a/src/frontend/src/api/ProjectTaskStatus.js +++ b/src/frontend/src/api/ProjectTaskStatus.js @@ -30,7 +30,7 @@ const UpdateTaskStatus = (url, style, existingData, currentProjectId, feature, m dispatch( HomeActions.SetSnackBar({ open: true, - message: `Task #${response.data.id} has been updated to ${response.data.task_status_str}`, + message: `Task #${response.data.id} has been updated to ${response.data.task_status}`, variant: 'success', duration: 3000, }), diff --git a/src/frontend/src/components/ActivitiesPanel.jsx b/src/frontend/src/components/ActivitiesPanel.jsx index f2ab3d369a..2725bad4d8 100755 --- a/src/frontend/src/components/ActivitiesPanel.jsx +++ b/src/frontend/src/components/ActivitiesPanel.jsx @@ -74,7 +74,7 @@ const ActivitiesPanel = ({ defaultTheme, state, params, map, view, mapDivPostion state[index].taskBoundries.forEach((task) => { taskHistories = taskHistories.concat( task.task_history.map((history) => { - return { ...history, taskId: task.id, status: task.task_status_str }; + return { ...history, taskId: task.id, status: task.task_status }; }), ); }); diff --git a/src/frontend/src/components/DialogTaskActions.jsx b/src/frontend/src/components/DialogTaskActions.jsx index 245c65d7a6..ee3258cff6 100755 --- a/src/frontend/src/components/DialogTaskActions.jsx +++ b/src/frontend/src/components/DialogTaskActions.jsx @@ -32,7 +32,7 @@ export default function Dialog({ taskId, feature, map, view }) { })[0], }; const findCorrectTaskStatusIndex = environment.tasksStatus.findIndex( - (data) => data.label == currentStatus.task_status_str, + (data) => data.label == currentStatus.task_status, ); const tasksStatus = feature.id_ != undefined ? environment.tasksStatus[findCorrectTaskStatusIndex]?.['label'] : ''; diff --git a/src/frontend/src/components/TasksLayer.jsx b/src/frontend/src/components/TasksLayer.jsx index a0542bd281..643fdc7b32 100755 --- a/src/frontend/src/components/TasksLayer.jsx +++ b/src/frontend/src/components/TasksLayer.jsx @@ -29,7 +29,7 @@ const TasksLayer = (map, view, feature) => { geojsonObject['features'] = []; state.projectTaskBoundries[index].taskBoundries.forEach((task) => { geojsonObject['features'].push({ - id: `${task.id}_${task.task_status_str}`, + id: `${task.id}_${task.task_status}`, type: task.outline_geojson.type, geometry: task.outline_geojson.geometry, properties: { centroid: task.bbox }, diff --git a/src/frontend/src/components/createnewproject/SelectForm.tsx b/src/frontend/src/components/createnewproject/SelectForm.tsx index fb3ba7cafc..cb762e6e95 100644 --- a/src/frontend/src/components/createnewproject/SelectForm.tsx +++ b/src/frontend/src/components/createnewproject/SelectForm.tsx @@ -14,7 +14,7 @@ import { FormCategoryService, ValidateCustomForm } from '../../api/CreateProject import NewDefineAreaMap from '../../views/NewDefineAreaMap'; const osmFeatureTypeOptions = [ - { name: 'form_ways', value: 'existing_form', label: 'Use Existing Form' }, + { name: 'form_ways', value: 'existing_form', label: 'Use Existing Category' }, { name: 'form_ways', value: 'custom_form', label: 'Upload a Custom Form' }, ]; // @ts-ignore @@ -74,8 +74,15 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) =>
Select Form

- You may choose to either use the default XLS form defined in FMTM or upload a custom XLS form. - You may learn more about XLSforms here.{' '} + You may choose an existing category or upload a custom XLS form. + + {' '} + You may learn more about XLSforms{' '} + + here + + . + {' '}

@@ -98,7 +105,7 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) => />
= ({ const projectDetails = useAppSelector((state) => state.createproject.projectDetails); // //we use use-selector from redux to get all state of projectDetails from createProject slice - const selectFormWaysList = ['Use Existing Form', 'Upload a Custom Form']; + const selectFormWaysList = ['Use Existing Category', 'Upload a Custom Form']; const selectFormWays = selectFormWaysList.map((item) => ({ label: item, value: item })); const userDetails: any = CoreModules.useAppSelector((state) => state.login.loginToken); // //we use use-selector from redux to get all state of loginToken from login slice diff --git a/src/frontend/src/views/NewProjectDetails.jsx b/src/frontend/src/views/NewProjectDetails.jsx index 373ce3a38f..2f59ad4eea 100644 --- a/src/frontend/src/views/NewProjectDetails.jsx +++ b/src/frontend/src/views/NewProjectDetails.jsx @@ -122,7 +122,7 @@ const Home = () => { ...feature.outline_geojson.properties, centroid: feature.bbox, }, - id: `${feature.project_task_name}_${feature.task_status_str}`, + id: `${feature.project_task_name}_${feature.task_status}`, })); const taskBuildingGeojsonFeatureCollection = { ...geojsonObjectModel,