From 6083f7c9945a8bb61073e67ac255fc0fffce7357 Mon Sep 17 00:00:00 2001 From: Abhijit Gupta Date: Sat, 25 Nov 2023 13:41:43 -0500 Subject: [PATCH] All Time Cards (#238) --- backend/src/aggregation/layer1/user.py | 6 ++--- backend/src/aggregation/layer2/user.py | 27 +++++++++++++------ .../github/graphql/user/contribs/contribs.py | 2 -- backend/src/models/background.py | 3 +++ backend/src/processing/auth.py | 4 +++ backend/src/processing/user/svg.py | 9 ++++--- backend/src/processing/wrapped/main.py | 8 +++++- backend/src/render/template.py | 6 ++++- backend/src/render/top_langs.py | 5 +++- backend/src/render/top_repos.py | 13 ++++++--- backend/src/routers/assets/assets.py | 4 +-- backend/src/routers/background.py | 27 ++++++++++++++++++- backend/src/routers/users/main.py | 2 +- backend/src/routers/users/svg.py | 6 +++-- backend/src/utils/utils.py | 2 +- .../src/components/Home/DateRangeSection.js | 10 +++---- frontend/src/pages/Home/stages/Customize.js | 1 + 17 files changed, 97 insertions(+), 38 deletions(-) diff --git a/backend/src/aggregation/layer1/user.py b/backend/src/aggregation/layer1/user.py index f525cdc6..2a88c627 100644 --- a/backend/src/aggregation/layer1/user.py +++ b/backend/src/aggregation/layer1/user.py @@ -64,7 +64,6 @@ async def query_user_month( return user_month -# NOTE: can only be called once every 1-2 minutes from publisher due to separate alru_cache @alru_cache(ttl=timedelta(hours=6)) async def query_user( user_id: str, @@ -72,9 +71,10 @@ async def query_user( private_access: bool = False, start_date: date = date.today() - timedelta(365), end_date: date = date.today(), + max_time: int = 3600, # seconds no_cache: bool = False, ) -> Tuple[bool, UserPackage]: - # Return (possibly incomplete) within 45 seconds + # Return (possibly incomplete) within max_time seconds start_time = datetime.now() incomplete = False @@ -97,7 +97,7 @@ async def query_user( # Start with complete months and add any incomplete months all_user_packages: List[UserPackage] = [x.data for x in curr_data if x.complete] for month in new_months: - if datetime.now() - start_time < timedelta(seconds=40): + if datetime.now() - start_time < timedelta(seconds=max_time): temp = await query_user_month(user_id, access_token, private_access, month) if temp is not None: all_user_packages.append(temp.data) diff --git a/backend/src/aggregation/layer2/user.py b/backend/src/aggregation/layer2/user.py index 969c3fe0..75fe831b 100644 --- a/backend/src/aggregation/layer2/user.py +++ b/backend/src/aggregation/layer2/user.py @@ -14,17 +14,22 @@ async def _get_user( user_id: str, private_access: bool, start_date: date, end_date: date -) -> Optional[UserPackage]: +) -> Tuple[Optional[UserPackage], bool]: user_months = await get_user_months(user_id, private_access, start_date, end_date) if len(user_months) == 0: - return None + return None, False + + expected_num_months = ( + (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month) + 1 + ) + complete = len(user_months) == expected_num_months user_data = user_months[0].data for user_month in user_months[1:]: user_data += user_month.data # TODO: handle timezone_str here - return user_data.trim(start_date, end_date) + return user_data.trim(start_date, end_date), complete @alru_cache() @@ -33,17 +38,23 @@ async def get_user( start_date: date, end_date: date, no_cache: bool = False, -) -> Tuple[bool, Tuple[Optional[UserPackage], Optional[UpdateUserBackgroundTask]]]: +) -> Tuple[ + bool, Tuple[Optional[UserPackage], bool, Optional[UpdateUserBackgroundTask]] +]: user: Optional[PublicUserModel] = await db_get_public_user(user_id) if user is None: - return (False, (None, None)) + return (False, (None, False, None)) private_access = user.private_access or False - user_data = await _get_user(user_id, private_access, start_date, end_date) + user_data, complete = await _get_user(user_id, private_access, start_date, end_date) background_task = UpdateUserBackgroundTask( - user_id=user_id, access_token=user.access_token, private_access=private_access + user_id=user_id, + access_token=user.access_token, + private_access=private_access, + start_date=start_date, + end_date=end_date, ) - return (user_data is not None, (user_data, background_task)) + return (complete, (user_data, complete, background_task)) @alru_cache(ttl=timedelta(minutes=15)) diff --git a/backend/src/data/github/graphql/user/contribs/contribs.py b/backend/src/data/github/graphql/user/contribs/contribs.py index 87d24d0f..d2e33c8f 100644 --- a/backend/src/data/github/graphql/user/contribs/contribs.py +++ b/backend/src/data/github/graphql/user/contribs/contribs.py @@ -55,8 +55,6 @@ def get_user_contribution_events( access_token: Optional[str] = None, ) -> RawEvents: """Fetches user contributions (commits, issues, prs, reviews)""" - if (end_date - start_date).days > 365: - raise ValueError("date range can be at most 1 year") query = { "variables": { "login": user_id, diff --git a/backend/src/models/background.py b/backend/src/models/background.py index e6a3dc38..3676af09 100644 --- a/backend/src/models/background.py +++ b/backend/src/models/background.py @@ -1,3 +1,4 @@ +from datetime import date from typing import Optional from pydantic import BaseModel @@ -7,3 +8,5 @@ class UpdateUserBackgroundTask(BaseModel): user_id: str access_token: Optional[str] private_access: bool + start_date: Optional[date] + end_date: Optional[date] diff --git a/backend/src/processing/auth.py b/backend/src/processing/auth.py index 69847d04..c014772a 100644 --- a/backend/src/processing/auth.py +++ b/backend/src/processing/auth.py @@ -46,6 +46,8 @@ async def authenticate( user_id=user_id, access_token=access_token, private_access=new_private_access, + start_date=None, + end_date=None, ) else: # first time sign up @@ -53,6 +55,8 @@ async def authenticate( user_id=user_id, access_token=access_token, private_access=private_access, + start_date=None, + end_date=None, ) await db_update_user(user_id, raw_user) diff --git a/backend/src/processing/user/svg.py b/backend/src/processing/user/svg.py index 232de21d..6a3ee8af 100644 --- a/backend/src/processing/user/svg.py +++ b/backend/src/processing/user/svg.py @@ -14,18 +14,19 @@ async def svg_base( time_range: str, demo: bool, no_cache: bool = False, -) -> Tuple[Optional[UserPackage], Optional[UpdateUserBackgroundTask], str]: +) -> Tuple[Optional[UserPackage], bool, Optional[UpdateUserBackgroundTask], str]: # process time_range, start_date, end_date time_range = "one_month" if demo else time_range start_date, end_date, time_str = use_time_range(time_range, start_date, end_date) + complete = True # overwritten later if not complete + background_task = None # fetch data, either using demo or user method - background_task = None if demo: output = await get_user_demo(user_id, start_date, end_date, no_cache=no_cache) else: - output, background_task = await get_user( + output, complete, background_task = await get_user( user_id, start_date, end_date, no_cache=no_cache ) - return output, background_task, time_str + return output, complete, background_task, time_str diff --git a/backend/src/processing/wrapped/main.py b/backend/src/processing/wrapped/main.py index 20122693..f3f4900d 100644 --- a/backend/src/processing/wrapped/main.py +++ b/backend/src/processing/wrapped/main.py @@ -17,7 +17,13 @@ async def query_wrapped_user( access_token = None if user is None else user.access_token private_access = False if user is None else user.private_access or False user_package: UserPackage = await query_user( - user_id, access_token, private_access, start_date, end_date, no_cache=True + user_id, + access_token, + private_access, + start_date, + end_date, + max_time=40, + no_cache=True, ) wrapped_package = get_wrapped_data(user_package, year) diff --git a/backend/src/render/template.py b/backend/src/render/template.py index 8eb76769..7b5ab1d9 100644 --- a/backend/src/render/template.py +++ b/backend/src/render/template.py @@ -79,9 +79,13 @@ def get_bar_section( ) ) total_percent, total_items = 0, len(data_row) + diff = max(0, 300 / bar_width - data_row[-1][0]) for j, (percent, color) in enumerate(data_row): color = color or DEFAULT_COLOR - percent = max(300 / bar_width, percent) + if j == 0: + percent -= diff + elif j == total_items - 1: + percent += diff bar_percent = bar_width * percent / 100 bar_total = bar_width * total_percent / 100 box_size, insert = (bar_percent, 8), (bar_total, 0) diff --git a/backend/src/render/top_langs.py b/backend/src/render/top_langs.py index c9afc1e6..cec169e8 100644 --- a/backend/src/render/top_langs.py +++ b/backend/src/render/top_langs.py @@ -15,6 +15,7 @@ def get_top_langs_svg( time_str: str, use_percent: bool, loc_metric: str, + complete: bool, commits_excluded: int, compact: bool, use_animation: bool, @@ -24,7 +25,9 @@ def get_top_langs_svg( subheader = time_str if not use_percent: subheader += " | " + ("LOC Changed" if loc_metric == "changed" else "LOC Added") - if commits_excluded > 50: + if not complete: + subheader += " | Incomplete (refresh to update)" + elif commits_excluded > 50: subheader += f" | {commits_excluded} commits excluded" if len(data) <= 1: diff --git a/backend/src/render/top_repos.py b/backend/src/render/top_repos.py index 14eddcdc..08833364 100644 --- a/backend/src/render/top_repos.py +++ b/backend/src/render/top_repos.py @@ -1,5 +1,6 @@ # type: ignore +from collections import defaultdict from typing import List, Tuple from svgwrite import Drawing @@ -14,6 +15,7 @@ def get_top_repos_svg( data: List[RepoStats], time_str: str, loc_metric: str, + complete: bool, commits_excluded: int, use_animation: bool, theme: str, @@ -21,7 +23,9 @@ def get_top_repos_svg( header = "Most Contributed Repositories" subheader = time_str subheader += " | " + ("LOC Changed" if loc_metric == "changed" else "LOC Added") - if commits_excluded > 50: + if not complete: + subheader += " | Incomplete (refresh to update)" + elif commits_excluded > 50: subheader += f" | {commits_excluded} commits excluded" if len(data) == 0: @@ -53,11 +57,12 @@ def get_top_repos_svg( get_bar_section(d=d, dataset=dataset, theme=theme, padding=45, bar_width=195) ) - langs = {} + langs = defaultdict(int) for x in data[:4]: for lang in x.langs: - langs[lang.lang] = lang.color - langs = list(langs.items())[:6] + langs[(lang.lang, lang.color)] += lang.loc + langs = sorted(langs.items(), key=lambda x: x[1], reverse=True) + langs = [lang[0] for lang in langs[:6]] columns = {1: 1, 2: 2, 3: 3, 4: 2, 5: 3, 6: 3}[len(langs)] padding = 215 + (10 if columns == len(langs) else 0) diff --git a/backend/src/routers/assets/assets.py b/backend/src/routers/assets/assets.py index 4f994571..917dd45c 100644 --- a/backend/src/routers/assets/assets.py +++ b/backend/src/routers/assets/assets.py @@ -6,9 +6,9 @@ @router.get("/error", status_code=status.HTTP_200_OK, include_in_schema=False) async def get_error_img(): - return FileResponse("./src/publisher/routers/assets/assets/error.png") + return FileResponse("./src/routers/assets/assets/error.png") @router.get("/stopwatch", status_code=status.HTTP_200_OK, include_in_schema=False) async def get_stopwatch_img(): - return FileResponse("./src/publisher/routers/assets/assets/stopwatch.png") + return FileResponse("./src/routers/assets/assets/stopwatch.png") diff --git a/backend/src/routers/background.py b/backend/src/routers/background.py index 34d4f940..2892f02a 100644 --- a/backend/src/routers/background.py +++ b/backend/src/routers/background.py @@ -1,7 +1,32 @@ +from typing import Dict + from src.aggregation.layer1 import query_user from src.models.background import UpdateUserBackgroundTask +# create a cache for the function +cache: Dict[str, Dict[str, bool]] = {"update_user": {}} + async def run_in_background(task: UpdateUserBackgroundTask): if isinstance(task, UpdateUserBackgroundTask): # type: ignore - await query_user(task.user_id, task.access_token, task.private_access) + inputs = { + "user_id": task.user_id, + "access_token": task.access_token, + "private_access": task.private_access, + "start_date": task.start_date, + "end_date": task.end_date, + } + + inputs = {k: v for k, v in inputs.items() if v is not None} + + # check if the task is already running + if task.user_id in cache["update_user"]: + return + + # add the task to the cache + cache["update_user"][task.user_id] = True + + await query_user(**inputs) # type: ignore + + # remove the task from the cache + del cache["update_user"][task.user_id] diff --git a/backend/src/routers/users/main.py b/backend/src/routers/users/main.py index d52542ea..bf7f5fe2 100644 --- a/backend/src/routers/users/main.py +++ b/backend/src/routers/users/main.py @@ -31,7 +31,7 @@ async def get_user_endpoint( timezone_str: str = "US/Eastern", no_cache: bool = False, ) -> Optional[UserPackage]: - output, background_task = await get_user( + output, _, background_task = await get_user( user_id, start_date, end_date, no_cache=no_cache ) if background_task is not None: diff --git a/backend/src/routers/users/svg.py b/backend/src/routers/users/svg.py index c97f6a2b..2ad36875 100644 --- a/backend/src/routers/users/svg.py +++ b/backend/src/routers/users/svg.py @@ -39,7 +39,7 @@ async def get_user_lang_svg( use_animation: bool = True, theme: str = "classic", ) -> Any: - output, background_task, time_str = await svg_base( + output, complete, background_task, time_str = await svg_base( user_id, start_date, end_date, time_range, demo, no_cache ) if background_task is not None: @@ -57,6 +57,7 @@ async def get_user_lang_svg( time_str=time_str, use_percent=use_percent, loc_metric=loc_metric, + complete=complete, commits_excluded=commits_excluded, compact=compact, use_animation=use_animation, @@ -84,7 +85,7 @@ async def get_user_repo_svg( use_animation: bool = True, theme: str = "classic", ) -> Any: - output, background_task, time_str = await svg_base( + output, complete, background_task, time_str = await svg_base( user_id, start_date, end_date, time_range, demo, no_cache ) if background_task is not None: @@ -103,6 +104,7 @@ async def get_user_repo_svg( data=processed, time_str=time_str, loc_metric=loc_metric, + complete=complete, commits_excluded=commits_excluded, use_animation=use_animation, theme=theme, diff --git a/backend/src/utils/utils.py b/backend/src/utils/utils.py index 2ecfbc3d..969bb9d8 100644 --- a/backend/src/utils/utils.py +++ b/backend/src/utils/utils.py @@ -5,7 +5,6 @@ def date_to_datetime( dt: date, hour: int = 0, minute: int = 0, second: int = 0 ) -> datetime: - return datetime(dt.year, dt.month, dt.day, hour, minute, second) @@ -18,6 +17,7 @@ def use_time_range( "three_months": (90, "Past 3 Months"), "six_months": (180, "Past 6 Months"), "one_year": (365, "Past 1 Year"), + "all_time": (365 * 10, "All Time"), } start_str = start_date.strftime("X%m/X%d/%Y").replace("X0", "X").replace("X", "") diff --git a/frontend/src/components/Home/DateRangeSection.js b/frontend/src/components/Home/DateRangeSection.js index a7b1048c..a2ba3d24 100644 --- a/frontend/src/components/Home/DateRangeSection.js +++ b/frontend/src/components/Home/DateRangeSection.js @@ -7,7 +7,7 @@ import { Input } from '../Generic'; const DateRangeSection = ({ selectedTimeRange, setSelectedTimeRange, - disabled, + privateAccess, }) => { const timeRangeOptions = [ { id: 1, label: 'Past 1 Month', disabled: false, value: 'one_month' }, @@ -19,6 +19,7 @@ const DateRangeSection = ({ }, { id: 2, label: 'Past 6 Months', disabled: false, value: 'six_months' }, { id: 3, label: 'Past 1 Year', disabled: false, value: 'one_year' }, + { id: 4, label: 'All Time', disabled: !privateAccess, value: 'all_time' }, ]; const selectedOption = selectedTimeRange || timeRangeOptions[2]; @@ -30,7 +31,6 @@ const DateRangeSection = ({ options={timeRangeOptions} selectedOption={selectedOption} setSelectedOption={setSelectedTimeRange} - disabled={disabled} /> ); @@ -39,11 +39,7 @@ const DateRangeSection = ({ DateRangeSection.propTypes = { selectedTimeRange: PropTypes.object.isRequired, setSelectedTimeRange: PropTypes.func.isRequired, - disabled: PropTypes.bool, -}; - -DateRangeSection.defaultProps = { - disabled: false, + privateAccess: PropTypes.bool.isRequired, }; export default DateRangeSection; diff --git a/frontend/src/pages/Home/stages/Customize.js b/frontend/src/pages/Home/stages/Customize.js index 403aa938..574e6caa 100644 --- a/frontend/src/pages/Home/stages/Customize.js +++ b/frontend/src/pages/Home/stages/Customize.js @@ -28,6 +28,7 @@ const CustomizeStage = ({ {selectedCard === 'langs' && (