diff --git a/.gitignore b/.gitignore index 5bbbe537..1a817f76 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__ .DS_Store registered_log.txt schedule.json +pretix_cache.json diff --git a/EuroPythonBot/config.toml b/EuroPythonBot/config.toml index f1bcbbb2..e5123d30 100644 --- a/EuroPythonBot/config.toml +++ b/EuroPythonBot/config.toml @@ -37,6 +37,7 @@ REGISTERED_LOG_FILE = "registered_log.txt" [pretix] PRETIX_BASE_URL = "https://pretix.eu/api/v1/organizers/europython/events/ep2024" +PRETIX_CACHE_FILE = "pretix_cache.json" [logging] LOG_LEVEL = "INFO" diff --git a/EuroPythonBot/configuration.py b/EuroPythonBot/configuration.py index 21a9b194..73bfef81 100644 --- a/EuroPythonBot/configuration.py +++ b/EuroPythonBot/configuration.py @@ -41,6 +41,7 @@ def __init__(self): # Pretix self.PRETIX_BASE_URL = config["pretix"]["PRETIX_BASE_URL"] + self.PRETIX_CACHE_FILE = Path(config["pretix"]["PRETIX_CACHE_FILE"]) role_name_to_id: dict[str, int] = config["roles"] self.ITEM_TO_ROLES: dict[str, list[int]] = self._translate_role_names_to_ids( diff --git a/EuroPythonBot/registration/cog.py b/EuroPythonBot/registration/cog.py index 821cf938..ad179d5a 100644 --- a/EuroPythonBot/registration/cog.py +++ b/EuroPythonBot/registration/cog.py @@ -156,7 +156,9 @@ def __init__(self, bot: Client): self.bot = bot self.pretix_connector = PretixConnector( - url=config.PRETIX_BASE_URL, token=os.environ["PRETIX_TOKEN"] + url=config.PRETIX_BASE_URL, + token=os.environ["PRETIX_TOKEN"], + cache_file=config.PRETIX_CACHE_FILE, ) self.registration_logger = RegistrationLogger(config.REGISTERED_LOG_FILE) _logger.info("Cog 'Registration' has been initialized") diff --git a/EuroPythonBot/registration/pretix_connector.py b/EuroPythonBot/registration/pretix_connector.py index 1d4bb8ce..ddfe1a18 100644 --- a/EuroPythonBot/registration/pretix_connector.py +++ b/EuroPythonBot/registration/pretix_connector.py @@ -6,27 +6,51 @@ import time from collections import defaultdict from datetime import datetime, timedelta, timezone +from pathlib import Path +import aiofiles import aiohttp +from pydantic import BaseModel -from registration.pretix_api_response_models import PretixItem, PretixItemVariation, PretixOrder +from registration.pretix_api_response_models import PretixItem, PretixOrder from registration.ticket import Ticket, generate_ticket_key _logger = logging.getLogger(f"bot.{__name__}") +class PretixCache(BaseModel): + item_names_by_id: dict[int, str] + tickets_by_key: dict[str, list[Ticket]] + + class PretixConnector: - def __init__(self, *, url: str, token: str): + def __init__(self, *, url: str, token: str, cache_file: Path | None = None): self._pretix_api_url = url # https://docs.pretix.eu/en/latest/api/tokenauth.html#using-an-api-token self._http_headers = {"Authorization": f"Token {token}"} self._fetch_lock = asyncio.Lock() + self._last_fetch: datetime | None = None - self.items_by_id: dict[int, PretixItem | PretixItemVariation] = {} + self._cache_file = cache_file + + self.item_names_by_id: dict[int, str] = {} self.tickets_by_key: dict[str, list[Ticket]] = defaultdict(list) - self._last_fetch: datetime | None = None + + self._load_cache() + + def _load_cache(self) -> None: + if self._cache_file is None or not self._cache_file.exists(): + return # no cache configured, or file does not yet exist + + file_content = self._cache_file.read_bytes() + if not file_content: + return # file is empty, e.g. `touch`ed by ansible + + cache = PretixCache.model_validate_json(file_content) + self.item_names_by_id = cache.item_names_by_id + self.tickets_by_key = cache.tickets_by_key async def fetch_pretix_data(self) -> None: """Fetch order and item data from the Pretix API and cache it.""" @@ -40,6 +64,15 @@ async def fetch_pretix_data(self) -> None: await self._fetch_pretix_items() await self._fetch_pretix_orders(since=self._last_fetch) + + if self._cache_file is not None: + async with aiofiles.open(self._cache_file, "w") as f: + cache = PretixCache( + item_names_by_id=self.item_names_by_id, + tickets_by_key=self.tickets_by_key, + ) + await f.write(cache.model_dump_json()) + self._last_fetch = now async def _fetch_pretix_orders(self, since: datetime | None = None) -> None: @@ -64,12 +97,10 @@ async def _fetch_pretix_orders(self, since: datetime | None = None) -> None: if not position.attendee_name: continue - item = self.items_by_id[position.item_id] - item_name = item.names_by_locale["en"] + item_name = self.item_names_by_id[position.item_id] if position.variation_id is not None: - variation = self.items_by_id[position.variation_id] - variation_name = variation.names_by_locale["en"] + variation_name = self.item_names_by_id[position.variation_id] else: variation_name = None @@ -91,9 +122,9 @@ async def _fetch_pretix_items(self) -> None: for item_as_json in items_as_json: item = PretixItem(**item_as_json) - self.items_by_id[item.id] = item + self.item_names_by_id[item.id] = item.names_by_locale["en"] for variation in item.variations: - self.items_by_id[variation.id] = variation + self.item_names_by_id[variation.id] = variation.names_by_locale["en"] async def _fetch_all_pages(self, url: str, params: dict[str, str] | None = None) -> list[dict]: """Fetch all pages from a paginated Pretix API endpoint.""" diff --git a/EuroPythonBot/registration/ticket.py b/EuroPythonBot/registration/ticket.py index 239f5218..cd7de5c5 100644 --- a/EuroPythonBot/registration/ticket.py +++ b/EuroPythonBot/registration/ticket.py @@ -1,6 +1,6 @@ import string -from dataclasses import dataclass +from pydantic import BaseModel, ConfigDict, computed_field from unidecode import unidecode @@ -16,13 +16,14 @@ def generate_ticket_key(*, order: str, name: str) -> str: return f"{order}-{name}" -@dataclass(frozen=True) -class Ticket: +class Ticket(BaseModel): + model_config = ConfigDict(frozen=True) + order: str name: str type: str variation: str | None - @property + @computed_field def key(self) -> str: return generate_ticket_key(order=self.order, name=self.name) diff --git a/EuroPythonBot/staging-config.toml b/EuroPythonBot/staging-config.toml index b4d4d811..1d0cc867 100644 --- a/EuroPythonBot/staging-config.toml +++ b/EuroPythonBot/staging-config.toml @@ -34,6 +34,7 @@ REGISTERED_LOG_FILE = "registered_log.txt" [pretix] PRETIX_BASE_URL = "https://pretix.eu/api/v1/organizers/europython/events/ep2023-staging2" +PRETIX_CACHE_FILE = "pretix_cache.json" [logging] LOG_LEVEL = "INFO" @@ -43,6 +44,7 @@ LOG_LEVEL = "INFO" timezone_offset = 2 api_url = "https://programapi24.europython.eu/2024/schedule.json" schedule_cache_file = "schedule.json" +livestream_url_file = "livestreams.toml" # optional simulated start time for testing program notifications # simulated_start_time = "2024-07-10T07:30:00" diff --git a/ansible/deploy-playbook.yml b/ansible/deploy-playbook.yml index 457fa728..72b9ad6d 100644 --- a/ansible/deploy-playbook.yml +++ b/ansible/deploy-playbook.yml @@ -87,6 +87,13 @@ owner: bot group: bot + - name: Create pretix_cache.json in bot's home directory + file: + path: /home/bot/pretix_cache.json + state: touch + owner: bot + group: bot + - name: Create schedule.json in bot's home directory file: path: /home/bot/schedule.json diff --git a/compose.yaml b/compose.yaml index 86f88dc8..c9eb836f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -23,6 +23,11 @@ services: target: /home/bot/schedule.json read_only: false + - type: bind + source: /home/bot/pretix_cache.json + target: /home/bot/pretix_cache.json + read_only: false + # read all container only logs with # journalctl -u docker IMAGE_NAME=europythonbot -f logging: diff --git a/tests/registration/test_pretix_connector.py b/tests/registration/test_pretix_connector.py index 5963edbd..cbffcf82 100644 --- a/tests/registration/test_pretix_connector.py +++ b/tests/registration/test_pretix_connector.py @@ -86,15 +86,15 @@ async def test_pretix_items(pretix_mock): await pretix_connector.fetch_pretix_data() - items_by_id = pretix_connector.items_by_id + item_names_by_id = pretix_connector.item_names_by_id - assert len(items_by_id) == 5 + assert len(item_names_by_id) == 5 - assert items_by_id[339041].names_by_locale["en"] == "Business" - assert items_by_id[163246].names_by_locale["en"] == "Conference" - assert items_by_id[163247].names_by_locale["en"] == "Tutorials" - assert items_by_id[339042].names_by_locale["en"] == "Personal" - assert items_by_id[163253].names_by_locale["en"] == "Combined (Conference + Tutorials)" + assert item_names_by_id[339041] == "Business" + assert item_names_by_id[163246] == "Conference" + assert item_names_by_id[163247] == "Tutorials" + assert item_names_by_id[339042] == "Personal" + assert item_names_by_id[163253] == "Combined (Conference + Tutorials)" @pytest.mark.asyncio @@ -128,6 +128,25 @@ async def test_get_ticket(pretix_mock): ] +async def test_cache(pretix_mock, tmp_path): + pretix_connector_1 = PretixConnector( + url=pretix_mock.base_url, token=PRETIX_API_TOKEN, cache_file=tmp_path / "pretix_cache.json" + ) + assert not pretix_connector_1.item_names_by_id + assert not pretix_connector_1.tickets_by_key + + # fetch data in connector 1 + await pretix_connector_1.fetch_pretix_data() + assert pretix_connector_1.item_names_by_id + assert pretix_connector_1.tickets_by_key + + pretix_connector_2 = PretixConnector( + url=pretix_mock.base_url, token=PRETIX_API_TOKEN, cache_file=tmp_path / "pretix_cache.json" + ) + assert pretix_connector_1.item_names_by_id == pretix_connector_2.item_names_by_id + assert pretix_connector_1.tickets_by_key == pretix_connector_2.tickets_by_key + + async def test_get_ticket_handles_ticket_ids(pretix_mock): pretix_connector = PretixConnector(url=pretix_mock.base_url, token=PRETIX_API_TOKEN) @@ -272,7 +291,9 @@ async def test_pagination(aiohttp_client, unused_tcp_port_factory): pretix_connector = PretixConnector(url=pretix_mock.base_url, token=PRETIX_API_TOKEN) await pretix_connector.fetch_pretix_data() - assert len(pretix_connector.items_by_id) == 5, "Only the first page of '/items' was fetched." + assert ( + len(pretix_connector.item_names_by_id) == 5 + ), "Only the first page of '/items' was fetched." @pytest.mark.asyncio diff --git a/tests/registration/test_registration_logger.py b/tests/registration/test_registration_logger.py index f720ad66..708f0e0a 100644 --- a/tests/registration/test_registration_logger.py +++ b/tests/registration/test_registration_logger.py @@ -9,7 +9,9 @@ def test_with_empty_file(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - assert not logger.is_registered(Ticket("ABC01", "John Doe", "Business", "Tutorials")) + assert not logger.is_registered( + Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials") + ) def test_with_existing_file(tmp_path: Path) -> None: @@ -17,14 +19,16 @@ def test_with_existing_file(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - assert logger.is_registered(Ticket("ABC01", "John Doe", "Business", "Tutorials")) + assert logger.is_registered( + Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials") + ) @pytest.mark.asyncio async def test_register_ticket_on_empty_log(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - ticket = Ticket("ABC01", "John Doe", "Business", "Tutorials") + ticket = Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials") await logger.mark_as_registered(ticket) @@ -36,7 +40,7 @@ async def test_register_ticket_on_empty_log(tmp_path: Path) -> None: async def test_register_ticket_with_existing_file(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - ticket = Ticket("ABC01", "John Doe", "Business", "Tutorials") + ticket = Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials") await logger.mark_as_registered(ticket) @@ -50,7 +54,7 @@ async def test_register_ticket_with_existing_log(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - ticket = Ticket("ABC02", "Jane Doe", "Business", "Tutorials") + ticket = Ticket(order="ABC02", name="Jane Doe", type="Business", variation="Tutorials") await logger.mark_as_registered(ticket) @@ -62,7 +66,7 @@ async def test_register_ticket_with_existing_log(tmp_path: Path) -> None: async def test_register_already_registered_ticket(tmp_path: Path) -> None: logger = RegistrationLogger(tmp_path / "registrations.txt") - ticket = Ticket("ABC02", "Jane Doe", "Business", "Tutorials") + ticket = Ticket(order="ABC02", name="Jane Doe", type="Business", variation="Tutorials") await logger.mark_as_registered(ticket)