From 2c0f596e3e556863140beb02c3c1b2c92783aff4 Mon Sep 17 00:00:00 2001 From: Samuel Giffard Date: Sun, 4 Feb 2024 12:25:58 +0100 Subject: [PATCH] feat: JSON Schema for Events Signed-off-by: Samuel Giffard --- dvalin-tools/dvalin_tools/models/common.py | 14 +- dvalin-tools/dvalin_tools/scrapers/events.py | 61 +++- schemas/Events.json | 366 ++++++++++--------- 3 files changed, 247 insertions(+), 194 deletions(-) diff --git a/dvalin-tools/dvalin_tools/models/common.py b/dvalin-tools/dvalin_tools/models/common.py index 168d696332d..623cc702d09 100644 --- a/dvalin-tools/dvalin_tools/models/common.py +++ b/dvalin-tools/dvalin_tools/models/common.py @@ -1,8 +1,8 @@ from datetime import datetime from enum import Enum -from typing import Annotated, Any, Callable, Generic, TypeVar +from typing import Annotated, Any, Callable, Generic, TypeVar, cast -from pydantic import PlainSerializer, ValidationError, WrapValidator +from pydantic import PlainSerializer, ValidationError, WithJsonSchema, WrapValidator from pydantic_core.core_schema import ValidationInfo, ValidatorFunctionWrapHandler _EnumStrSerializer = PlainSerializer(lambda e: str(e.name), return_type=str) @@ -54,9 +54,15 @@ class EnumSerializeAndValidateAsStr(Generic[T]): def __class_getitem__( cls, item: type[T] - ) -> Annotated[T, PlainSerializer, WrapValidator]: + ) -> Annotated[T, PlainSerializer, WrapValidator, WithJsonSchema]: + enum_schema = { + "enum": [_EnumStrSerializer.func(m) for m in item.__members__.values()] + } return Annotated[ - item, _EnumStrSerializer, WrapValidator(accept_enum_names(item)) + item, + _EnumStrSerializer, + WrapValidator(accept_enum_names(item)), + WithJsonSchema(enum_schema), ] diff --git a/dvalin-tools/dvalin_tools/scrapers/events.py b/dvalin-tools/dvalin_tools/scrapers/events.py index e6c89682ea2..95dccd7b300 100644 --- a/dvalin-tools/dvalin_tools/scrapers/events.py +++ b/dvalin-tools/dvalin_tools/scrapers/events.py @@ -5,12 +5,13 @@ """ import asyncio +import json +from argparse import ArgumentParser, Namespace from asyncio import TaskGroup from datetime import datetime from enum import Flag, auto from itertools import count from pathlib import Path, PurePath -from argparse import ArgumentParser, Namespace import aiofiles import httpx @@ -22,7 +23,7 @@ from tqdm.asyncio import tqdm_asyncio from dvalin_tools.lib.common import batched -from dvalin_tools.lib.constants import DATA_DIR +from dvalin_tools.lib.constants import DATA_DIR, ROOT_DIR_DVALIN_DATA from dvalin_tools.lib.languages import LANGUAGE_CODE_TO_DIR, LanguageCode from dvalin_tools.lib.tags import get_tags_from_subject from dvalin_tools.models.common import Game @@ -306,6 +307,12 @@ async def update_all_event_files( await tqdm_asyncio.gather(*tasks) +def generate_json_schema(output: Path) -> None: + """Generate JSON schema for events.""" + schema = EventFile.model_json_schema() + output.write_text(json.dumps(schema, indent=2), encoding="utf-8") + + def get_arg_parser() -> ArgumentParser: parser = ArgumentParser(description="Run scraper for Genhin Impact events.") subparsers = parser.add_subparsers(dest="subcommand", required=True) @@ -313,7 +320,8 @@ def get_arg_parser() -> ArgumentParser: get_parser = subparsers.add_parser("get", help="Get events.") get_parser.add_argument( - "-l", "--limit", + "-l", + "--limit", type=int, default=25, help="Limit of events to get.", @@ -321,36 +329,63 @@ def get_arg_parser() -> ArgumentParser: subparsers.add_parser("reparse", help="Reparse event files.") - update_parser = subparsers.add_parser("update", help="Update event files.", - usage="\r\n".join(["The different modes do the following:", - "- DETAILS_DL: Download the details of the events.", - "- LINKS: Update the links of the events. It will attempt to resolve all the URLs mentioned in the content.", - "- RESOLVE_URLS: Resolve the URLs of the links of the events.", - "- IMAGES_DL: Download the images of the events.",])) + update_parser = subparsers.add_parser( + "update", + help="Update event files.", + usage="\r\n".join( + [ + "The different modes do the following:", + "- DETAILS_DL: Download the details of the events.", + "- LINKS: Update the links of the events. It will attempt to resolve all the URLs mentioned in the content.", + "- RESOLVE_URLS: Resolve the URLs of the links of the events.", + "- IMAGES_DL: Download the images of the events.", + ] + ), + ) update_parser.add_argument( - "-f", "--force", + "-f", + "--force", action="store_true", help="Force update, even if the data is already present.", ) update_parser.add_argument( - "-m", "--mode", + "-m", + "--mode", type=lambda x: UpdateMode[x.upper()], required=True, help=f"Update mode. Possible values: {', '.join(UpdateMode.__members__)}.", ) + schema_parser = subparsers.add_parser( + "schema", help="Generate JSON schema for events." + ) + + schema_parser.add_argument( + "-o", + "--output", + type=Path, + default=ROOT_DIR_DVALIN_DATA / "schemas" / "Events.json", + help="Output file.", + ) + return parser async def async_main(namespace: Namespace): if namespace.subcommand == "get": - events = await get_all_events(Game.GENSHIN_IMPACT, MessageType.INFO, limit=namespace.limit) + events = await get_all_events( + Game.GENSHIN_IMPACT, MessageType.INFO, limit=namespace.limit + ) write_events(events, DATA_DIR) elif namespace.subcommand == "reparse": reparse_event_files(DATA_DIR) elif namespace.subcommand == "update": - await update_all_event_files(DATA_DIR, force=namespace.force, mode=namespace.mode) + await update_all_event_files( + DATA_DIR, force=namespace.force, mode=namespace.mode + ) + elif namespace.subcommand == "schema": + generate_json_schema(namespace.output) def main(): diff --git a/schemas/Events.json b/schemas/Events.json index d91b78b20c9..25044ef105f 100644 --- a/schemas/Events.json +++ b/schemas/Events.json @@ -1,190 +1,202 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "list": { - "type": "array", - "items": { - "type": "object", - "properties": { - "list": { - "type": "array", - "items": { - "type": "object", - "properties": { - "ann_id": { - "type": "number" - }, - "title": { - "type": "string" - }, - "subtitle": { - "type": "string" - }, - "banner": { - "type": "string" - }, - "content": { - "type": "string" - }, - "type_label": { - "type": "string" - }, - "tag_label": { - "type": "string" - }, - "tag_icon": { - "type": "string" - }, - "login_alert": { - "type": "number" - }, - "lang": { - "type": "string" - }, - "start_time": { - "type": "string" - }, - "end_time": { - "type": "string" - }, - "type": { - "type": "number" - }, - "remind": { - "type": "number" - }, - "alert": { - "type": "number" - }, - "tag_start_time": { - "type": "string" - }, - "tag_end_time": { - "type": "string" - }, - "remind_ver": { - "type": "number" - }, - "has_content": { - "type": "boolean" - }, - "extra_remind": { - "type": "number" - }, - "tag_icon_hover": { - "type": "string" - } - }, - "required": [ - "ann_id", - "title", - "subtitle", - "banner", - "content", - "type_label", - "tag_label", - "tag_icon", - "login_alert", - "lang", - "start_time", - "end_time", - "type", - "remind", - "alert", - "tag_start_time", - "tag_end_time", - "remind_ver", - "has_content", - "extra_remind", - "tag_icon_hover" - ] - } - }, - "type_id": { - "type": "number" - }, - "type_label": { - "type": "string" - } - }, - "required": [ - "list", - "type_id", - "type_label" + "$defs": { + "EventLocalized": { + "properties": { + "post_id": { + "title": "Post Id", + "type": "string" + }, + "game_id": { + "enum": [ + "HONKAI_IMPACT_3RD", + "GENSHIN_IMPACT", + "TEARS_OF_THEMIS", + "HOYOLAB", + "HONKAI_STAR_RAIL", + "ZENLESS_ZONE_ZERO" + ] + }, + "message_type": { + "enum": [ + "NOTICES", + "EVENT", + "INFO" ] + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "tags": { + "items": { + "enum": [ + "NEW_CHARACTER_INTRO", + "VOICE_ARTIST_ANNOUNCEMENT", + "VERSION_EVENT_WISHES_NOTICE", + "VERSION_EVENT_NOTICES_COMPILATION", + "CHARACTER_DEMO", + "CHARACTER_TEASER", + "STORY_QUEST", + "VERSION_NEW_WEAPON", + "NEW_OUTFIT", + "OUTFIT_TEASER", + "NEW_CONTENTS_DISPLAY", + "WEB_EVENT_WALLPAPERS", + "VERSION_PREVIEW", + "EVENT", + "VERSION_TRAILER", + "OST_ALBUM", + "SPECIAL_PROGRAM_PREVIEW", + "DEVELOPERS_DISCUSSION", + "COLLECTED_MISCELLANY", + "VERSION_EVENTS_PREVIEW", + "EVENT_TEASER", + "VERSION_NEW_ARTIFACT", + "UPDATE_PREVIEW", + "UPDATE_DETAILS", + "LEY_LINE_OVERFLOW", + "WALLPAPERS", + "GENIUS_INVOKATION_TCG", + "CUTSCENE_ANIMATION", + "GENSHIN_CONCERT_2023", + "VERSION_PREVIEW_PAGE", + "MUSIC" + ] + }, + "title": "Tags", + "type": "array", + "uniqueItems": true + }, + "language": { + "$ref": "#/$defs/LanguageCode" + }, + "subject": { + "title": "Subject", + "type": "string" + }, + "content": { + "default": "", + "title": "Content", + "type": "string" + }, + "links": { + "items": { + "$ref": "#/$defs/Link" + }, + "title": "Links", + "type": "array", + "uniqueItems": true } }, - "total": { - "type": "number" - }, - "type_list": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "number" + "required": [ + "post_id", + "game_id", + "message_type", + "created_at", + "language", + "subject" + ], + "title": "EventLocalized", + "type": "object" + }, + "LanguageCode": { + "enum": [ + "th-th", + "zh-cn", + "zh-tw", + "ru-ru", + "id-id", + "ko-kr", + "vi-vn", + "it-it", + "ja-jp", + "pt-pt", + "tr-tr", + "en-us", + "de-de", + "fr-fr", + "es-es" + ], + "title": "LanguageCode", + "type": "string" + }, + "Link": { + "properties": { + "index": { + "anyOf": [ + { + "type": "integer" }, - "name": { + { + "type": "null" + } + ], + "default": null, + "title": "Index" + }, + "url": { + "title": "Url", + "type": "string" + }, + "url_original": { + "title": "Url Original", + "type": "string" + }, + "url_original_resolved": { + "$ref": "#/$defs/RedirectLinkChain" + }, + "url_local": { + "anyOf": [ + { "type": "string" }, - "mi18n_name": { - "type": "string" + { + "type": "null" } - }, - "required": [ - "id", - "name", - "mi18n_name" + ], + "default": null, + "title": "Url Local" + }, + "link_type": { + "default": 12, + "enum": [ + "IMAGE", + "HOYO_LINK", + "HOYOLAB", + "MIHOYO_HOYOVERSE", + "TWITTER", + "FACEBOOK", + "YOUTUBE", + "TWITCH", + "VK", + "TELEGRAM", + "RELATIVE", + "UNKNOWN", + "MALFORMED" ] } }, - "alert": { - "type": "boolean" - }, - "alert_id": { - "type": "number" - }, - "timezone": { - "type": "number" - }, - "t": { + "required": [ + "url", + "url_original" + ], + "title": "Link", + "type": "object" + }, + "RedirectLinkChain": { + "items": { "type": "string" }, - "pic_list": { - "type": "array", - "items": {} - }, - "pic_total": { - "type": "number" - }, - "pic_type_list": { - "type": "array", - "items": {} - }, - "pic_alert": { - "type": "boolean" - }, - "pic_alert_id": { - "type": "number" - }, - "static_sign": { - "type": "string" - } - }, - "required": [ - "list", - "total", - "type_list", - "alert", - "alert_id", - "timezone", - "t", - "pic_list", - "pic_total", - "pic_type_list", - "pic_alert", - "pic_alert_id", - "static_sign" - ] - } + "title": "RedirectLinkChain", + "type": "array" + } + }, + "description": "A file containing events.", + "items": { + "$ref": "#/$defs/EventLocalized" + }, + "title": "EventFile", + "type": "array", + "uniqueItems": true +} \ No newline at end of file