From 688bf3e05b2502f557deb248ba69d56da02584bf Mon Sep 17 00:00:00 2001 From: Abhijit Gupta Date: Thu, 23 Nov 2023 14:40:12 -0500 Subject: [PATCH] Background tasks --- backend/src/aggregation/layer2/__init__.py | 2 +- backend/src/aggregation/layer2/user.py | 19 +++++++---------- backend/src/models/background.py | 9 ++++++++ backend/src/processing/auth.py | 24 ++++++++++++++++------ backend/src/processing/user/svg.py | 10 ++++++--- backend/src/routers/auth/website.py | 14 ++++++++++--- backend/src/routers/background.py | 7 +++++++ backend/src/routers/users/main.py | 13 ++++++++++-- backend/src/routers/users/svg.py | 15 +++++++++++--- 9 files changed, 83 insertions(+), 30 deletions(-) create mode 100644 backend/src/models/background.py create mode 100644 backend/src/routers/background.py diff --git a/backend/src/aggregation/layer2/__init__.py b/backend/src/aggregation/layer2/__init__.py index 1ed2309f..13c2a315 100644 --- a/backend/src/aggregation/layer2/__init__.py +++ b/backend/src/aggregation/layer2/__init__.py @@ -1,4 +1,4 @@ from src.aggregation.layer2.auth import get_is_valid_user from src.aggregation.layer2.user import get_user, get_user_demo -__all__ = ["get_user", "get_user_demo", "get_is_valid_user"] +__all__ = ["get_is_valid_user", "get_user", "get_user_demo"] diff --git a/backend/src/aggregation/layer2/user.py b/backend/src/aggregation/layer2/user.py index 3b57fd19..969c3fe0 100644 --- a/backend/src/aggregation/layer2/user.py +++ b/backend/src/aggregation/layer2/user.py @@ -6,6 +6,7 @@ from src.data.mongo.user import PublicUserModel, get_public_user as db_get_public_user from src.data.mongo.user_months import get_user_months from src.models import UserPackage +from src.models.background import UpdateUserBackgroundTask from src.utils import alru_cache # Formerly the publisher, loads existing data here @@ -26,29 +27,23 @@ async def _get_user( return user_data.trim(start_date, end_date) -@alru_cache() -async def update_user( - user_id: str, access_token: str, private_access: bool -) -> Tuple[bool, bool]: - # TODO: implement this - return (True, True) - - @alru_cache() async def get_user( user_id: str, start_date: date, end_date: date, no_cache: bool = False, -) -> Tuple[bool, Optional[UserPackage]]: +) -> Tuple[bool, Tuple[Optional[UserPackage], Optional[UpdateUserBackgroundTask]]]: user: Optional[PublicUserModel] = await db_get_public_user(user_id) if user is None: - return (False, None) + return (False, (None, None)) private_access = user.private_access or False - await update_user(user_id, user.access_token, private_access) user_data = await _get_user(user_id, private_access, start_date, end_date) - return (user_data is not None, user_data) + background_task = UpdateUserBackgroundTask( + user_id=user_id, access_token=user.access_token, private_access=private_access + ) + return (user_data is not None, (user_data, background_task)) @alru_cache(ttl=timedelta(minutes=15)) diff --git a/backend/src/models/background.py b/backend/src/models/background.py new file mode 100644 index 00000000..e6a3dc38 --- /dev/null +++ b/backend/src/models/background.py @@ -0,0 +1,9 @@ +from typing import Optional + +from pydantic import BaseModel + + +class UpdateUserBackgroundTask(BaseModel): + user_id: str + access_token: Optional[str] + private_access: bool diff --git a/backend/src/processing/auth.py b/backend/src/processing/auth.py index a99f649d..69847d04 100644 --- a/backend/src/processing/auth.py +++ b/backend/src/processing/auth.py @@ -1,6 +1,5 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple -from src.aggregation.layer2.user import update_user from src.data.github.auth import authenticate as github_authenticate from src.data.mongo.user import ( PublicUserModel, @@ -8,6 +7,7 @@ get_public_user as db_get_public_user, update_user as db_update_user, ) +from src.models.background import UpdateUserBackgroundTask # frontend first calls set_user_key with code and user_key # next they call authenticate which determines the user_id to associate with the code/user_key @@ -20,7 +20,9 @@ async def set_user_key(code: str, user_key: str) -> str: return user_key -async def authenticate(code: str, private_access: bool) -> str: +async def authenticate( + code: str, private_access: bool +) -> Tuple[str, Optional[UpdateUserBackgroundTask]]: user_id, access_token = await github_authenticate(code) curr_user: Optional[PublicUserModel] = await db_get_public_user(user_id) @@ -32,19 +34,29 @@ async def authenticate(code: str, private_access: bool) -> str: "private_access": private_access, } + background_task = None if curr_user is not None: curr_private_access = curr_user.private_access new_private_access = curr_private_access or private_access raw_user["private_access"] = new_private_access if new_private_access != curr_private_access: - await update_user(user_id, access_token, new_private_access) + # new private access status + background_task = UpdateUserBackgroundTask( + user_id=user_id, + access_token=access_token, + private_access=new_private_access, + ) else: # first time sign up - await update_user(user_id, access_token, private_access) + background_task = UpdateUserBackgroundTask( + user_id=user_id, + access_token=access_token, + private_access=private_access, + ) await db_update_user(user_id, raw_user) - return user_id + return user_id, background_task async def delete_user(user_id: str, user_key: str, use_user_key: bool = True) -> bool: diff --git a/backend/src/processing/user/svg.py b/backend/src/processing/user/svg.py index 06c5395d..232de21d 100644 --- a/backend/src/processing/user/svg.py +++ b/backend/src/processing/user/svg.py @@ -3,6 +3,7 @@ from src.aggregation.layer2.user import get_user, get_user_demo from src.models import UserPackage +from src.models.background import UpdateUserBackgroundTask from src.utils import use_time_range @@ -13,15 +14,18 @@ async def svg_base( time_range: str, demo: bool, no_cache: bool = False, -) -> Tuple[Optional[UserPackage], str]: +) -> Tuple[Optional[UserPackage], Optional[UpdateUserBackgroundTask], str]: # process time_range, start_date, end_date time_range = "one_month" if demo else time_range start_date, end_date, time_str = use_time_range(time_range, start_date, end_date) # fetch data, either using demo or user method + background_task = None if demo: output = await get_user_demo(user_id, start_date, end_date, no_cache=no_cache) else: - output = await get_user(user_id, start_date, end_date, no_cache=no_cache) + output, background_task = await get_user( + user_id, start_date, end_date, no_cache=no_cache + ) - return output, time_str + return output, background_task, time_str diff --git a/backend/src/routers/auth/website.py b/backend/src/routers/auth/website.py index 0be08baf..37b53211 100644 --- a/backend/src/routers/auth/website.py +++ b/backend/src/routers/auth/website.py @@ -1,10 +1,11 @@ from typing import Any, Dict -from fastapi import status +from fastapi import BackgroundTasks, status from fastapi.responses import Response from fastapi.routing import APIRouter from src.processing.auth import authenticate, delete_user, set_user_key +from src.routers.background import run_in_background from src.utils import async_fail_gracefully router = APIRouter() @@ -29,9 +30,16 @@ async def set_user_key_endpoint(response: Response, code: str, user_key: str) -> ) @async_fail_gracefully async def authenticate_endpoint( - response: Response, code: str, private_access: bool = False + response: Response, + background_tasks: BackgroundTasks, + code: str, + private_access: bool = False, ) -> str: - return await authenticate(code, private_access) + output, background_task = await authenticate(code, private_access) + if background_task is not None: + # set a background task to update the user + background_tasks.add_task(run_in_background, task=background_task) + return output @router.get( diff --git a/backend/src/routers/background.py b/backend/src/routers/background.py new file mode 100644 index 00000000..34d4f940 --- /dev/null +++ b/backend/src/routers/background.py @@ -0,0 +1,7 @@ +from src.aggregation.layer1 import query_user +from src.models.background import UpdateUserBackgroundTask + + +async def run_in_background(task: UpdateUserBackgroundTask): + if isinstance(task, UpdateUserBackgroundTask): # type: ignore + await query_user(task.user_id, task.access_token, task.private_access) diff --git a/backend/src/routers/users/main.py b/backend/src/routers/users/main.py index 0ea70cc6..d52542ea 100644 --- a/backend/src/routers/users/main.py +++ b/backend/src/routers/users/main.py @@ -1,10 +1,11 @@ from datetime import date, timedelta from typing import Any, Dict, Optional -from fastapi import APIRouter, Response, status +from fastapi import APIRouter, BackgroundTasks, Response, status from src.aggregation.layer2 import get_user from src.models import UserPackage +from src.routers.background import run_in_background from src.routers.users.db import router as db_router from src.routers.users.svg import router as svg_router from src.utils import async_fail_gracefully @@ -23,10 +24,18 @@ @async_fail_gracefully async def get_user_endpoint( response: Response, + background_tasks: BackgroundTasks, user_id: str, start_date: date = date.today() - timedelta(365), end_date: date = date.today(), timezone_str: str = "US/Eastern", no_cache: bool = False, ) -> Optional[UserPackage]: - return await get_user(user_id, start_date, end_date, no_cache=no_cache) + output, background_task = await get_user( + user_id, start_date, end_date, no_cache=no_cache + ) + if background_task is not None: + # set a background task to update the user + background_tasks.add_task(run_in_background, task=background_task) + + return output diff --git a/backend/src/routers/users/svg.py b/backend/src/routers/users/svg.py index d9d130a8..c97f6a2b 100644 --- a/backend/src/routers/users/svg.py +++ b/backend/src/routers/users/svg.py @@ -1,7 +1,7 @@ from datetime import date, timedelta from typing import Any -from fastapi import Response, status +from fastapi import BackgroundTasks, Response, status from fastapi.responses import HTMLResponse from fastapi.routing import APIRouter @@ -12,6 +12,7 @@ get_top_langs_svg, get_top_repos_svg, ) +from src.routers.background import run_in_background from src.routers.decorators import svg_fail_gracefully router = APIRouter() @@ -23,6 +24,7 @@ @svg_fail_gracefully async def get_user_lang_svg( response: Response, + background_tasks: BackgroundTasks, user_id: str, start_date: date = date.today() - timedelta(30), end_date: date = date.today(), @@ -37,9 +39,12 @@ async def get_user_lang_svg( use_animation: bool = True, theme: str = "classic", ) -> Any: - output, time_str = await svg_base( + output, background_task, time_str = await svg_base( user_id, start_date, end_date, time_range, demo, no_cache ) + if background_task is not None: + # set a background task to update the user + background_tasks.add_task(run_in_background, task=background_task) # if no data, return loading svg if output is None: @@ -65,6 +70,7 @@ async def get_user_lang_svg( @svg_fail_gracefully async def get_user_repo_svg( response: Response, + background_tasks: BackgroundTasks, user_id: str, start_date: date = date.today() - timedelta(30), end_date: date = date.today(), @@ -78,9 +84,12 @@ async def get_user_repo_svg( use_animation: bool = True, theme: str = "classic", ) -> Any: - output, time_str = await svg_base( + output, background_task, time_str = await svg_base( user_id, start_date, end_date, time_range, demo, no_cache ) + if background_task is not None: + # set a background task to update the user + background_tasks.add_task(run_in_background, task=background_task) # if no data, return loading svg if output is None: