Cache pretix data locally (#165)

* Move`Ticket` from dataclass to pydantic.BaseModel

* Cache pretix data

* Add pretix cache file to .gitignore

* Fix: Add livestreams file to staging-config.toml

* Add pretix cache file to docker-compose

* Fix typo

.gitignore

@@ -8,3 +8,4 @@ __pycache__
 .DS_Store
 registered_log.txt
 schedule.json
+pretix_cache.json

EuroPythonBot/config.toml

@@ -37,6 +37,7 @@ REGISTERED_LOG_FILE = "registered_log.txt"
 
 [pretix]
 PRETIX_BASE_URL = "https://pretix.eu/api/v1/organizers/europython/events/ep2024"
+PRETIX_CACHE_FILE = "pretix_cache.json"
 
 [logging]
 LOG_LEVEL = "INFO"

EuroPythonBot/configuration.py

@@ -41,6 +41,7 @@ def __init__(self):
 
             # Pretix
             self.PRETIX_BASE_URL = config["pretix"]["PRETIX_BASE_URL"]
+            self.PRETIX_CACHE_FILE = Path(config["pretix"]["PRETIX_CACHE_FILE"])
 
             role_name_to_id: dict[str, int] = config["roles"]
             self.ITEM_TO_ROLES: dict[str, list[int]] = self._translate_role_names_to_ids(

EuroPythonBot/registration/cog.py

@@ -156,7 +156,9 @@ def __init__(self, bot: Client):
         self.bot = bot
 
         self.pretix_connector = PretixConnector(
-            url=config.PRETIX_BASE_URL, token=os.environ["PRETIX_TOKEN"]
+            url=config.PRETIX_BASE_URL,
+            token=os.environ["PRETIX_TOKEN"],
+            cache_file=config.PRETIX_CACHE_FILE,
         )
         self.registration_logger = RegistrationLogger(config.REGISTERED_LOG_FILE)
         _logger.info("Cog 'Registration' has been initialized")

EuroPythonBot/registration/pretix_connector.py

@@ -6,27 +6,51 @@
 import time
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 
+import aiofiles
 import aiohttp
+from pydantic import BaseModel
 
-from registration.pretix_api_response_models import PretixItem, PretixItemVariation, PretixOrder
+from registration.pretix_api_response_models import PretixItem, PretixOrder
 from registration.ticket import Ticket, generate_ticket_key
 
 _logger = logging.getLogger(f"bot.{__name__}")
 
 
+class PretixCache(BaseModel):
+    item_names_by_id: dict[int, str]
+    tickets_by_key: dict[str, list[Ticket]]
+
+
 class PretixConnector:
-    def __init__(self, *, url: str, token: str):
+    def __init__(self, *, url: str, token: str, cache_file: Path | None = None):
         self._pretix_api_url = url
 
         # https://docs.pretix.eu/en/latest/api/tokenauth.html#using-an-api-token
         self._http_headers = {"Authorization": f"Token {token}"}
 
         self._fetch_lock = asyncio.Lock()
+        self._last_fetch: datetime | None = None
 
-        self.items_by_id: dict[int, PretixItem | PretixItemVariation] = {}
+        self._cache_file = cache_file
+
+        self.item_names_by_id: dict[int, str] = {}
         self.tickets_by_key: dict[str, list[Ticket]] = defaultdict(list)
-        self._last_fetch: datetime | None = None
+
+        self._load_cache()
+
+    def _load_cache(self) -> None:
+        if self._cache_file is None or not self._cache_file.exists():
+            return  # no cache configured, or file does not yet exist
+
+        file_content = self._cache_file.read_bytes()
+        if not file_content:
+            return  # file is empty, e.g. `touch`ed by ansible
+
+        cache = PretixCache.model_validate_json(file_content)
+        self.item_names_by_id = cache.item_names_by_id
+        self.tickets_by_key = cache.tickets_by_key
 
     async def fetch_pretix_data(self) -> None:
         """Fetch order and item data from the Pretix API and cache it."""
@@ -40,6 +64,15 @@ async def fetch_pretix_data(self) -> None:
 
             await self._fetch_pretix_items()
             await self._fetch_pretix_orders(since=self._last_fetch)
+
+            if self._cache_file is not None:
+                async with aiofiles.open(self._cache_file, "w") as f:
+                    cache = PretixCache(
+                        item_names_by_id=self.item_names_by_id,
+                        tickets_by_key=self.tickets_by_key,
+                    )
+                    await f.write(cache.model_dump_json())
+
             self._last_fetch = now
 
     async def _fetch_pretix_orders(self, since: datetime | None = None) -> None:
@@ -64,12 +97,10 @@ async def _fetch_pretix_orders(self, since: datetime | None = None) -> None:
                 if not position.attendee_name:
                     continue
 
-                item = self.items_by_id[position.item_id]
-                item_name = item.names_by_locale["en"]
+                item_name = self.item_names_by_id[position.item_id]
 
                 if position.variation_id is not None:
-                    variation = self.items_by_id[position.variation_id]
-                    variation_name = variation.names_by_locale["en"]
+                    variation_name = self.item_names_by_id[position.variation_id]
                 else:
                     variation_name = None
 
@@ -91,9 +122,9 @@ async def _fetch_pretix_items(self) -> None:
 
         for item_as_json in items_as_json:
             item = PretixItem(**item_as_json)
-            self.items_by_id[item.id] = item
+            self.item_names_by_id[item.id] = item.names_by_locale["en"]
             for variation in item.variations:
-                self.items_by_id[variation.id] = variation
+                self.item_names_by_id[variation.id] = variation.names_by_locale["en"]
 
     async def _fetch_all_pages(self, url: str, params: dict[str, str] | None = None) -> list[dict]:
         """Fetch all pages from a paginated Pretix API endpoint."""

EuroPythonBot/registration/ticket.py

@@ -1,6 +1,6 @@
 import string
-from dataclasses import dataclass
 
+from pydantic import BaseModel, ConfigDict, computed_field
 from unidecode import unidecode
 
 
@@ -16,13 +16,14 @@ def generate_ticket_key(*, order: str, name: str) -> str:
     return f"{order}-{name}"
 
 
-@dataclass(frozen=True)
-class Ticket:
+class Ticket(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
     order: str
     name: str
     type: str
     variation: str | None
 
-    @property
+    @computed_field
     def key(self) -> str:
         return generate_ticket_key(order=self.order, name=self.name)

EuroPythonBot/staging-config.toml

@@ -34,6 +34,7 @@ REGISTERED_LOG_FILE = "registered_log.txt"
 
 [pretix]
 PRETIX_BASE_URL = "https://pretix.eu/api/v1/organizers/europython/events/ep2023-staging2"
+PRETIX_CACHE_FILE = "pretix_cache.json"
 
 [logging]
 LOG_LEVEL = "INFO"
@@ -43,6 +44,7 @@ LOG_LEVEL = "INFO"
 timezone_offset = 2
 api_url = "https://programapi24.europython.eu/2024/schedule.json"
 schedule_cache_file = "schedule.json"
+livestream_url_file = "livestreams.toml"
 
 # optional simulated start time for testing program notifications
 # simulated_start_time = "2024-07-10T07:30:00"

ansible/deploy-playbook.yml

@@ -87,6 +87,13 @@
         owner: bot
         group: bot
 
+    - name: Create pretix_cache.json in bot's home directory
+      file:
+        path: /home/bot/pretix_cache.json
+        state: touch
+        owner: bot
+        group: bot
+
     - name: Create schedule.json in bot's home directory
       file:
         path: /home/bot/schedule.json

compose.yaml

@@ -23,6 +23,11 @@ services:
         target: /home/bot/schedule.json
         read_only: false
 
+      - type: bind
+        source: /home/bot/pretix_cache.json
+        target: /home/bot/pretix_cache.json
+        read_only: false
+
     # read all container only logs with
     # journalctl -u docker IMAGE_NAME=europythonbot -f
     logging:

tests/registration/test_pretix_connector.py

@@ -86,15 +86,15 @@ async def test_pretix_items(pretix_mock):
 
     await pretix_connector.fetch_pretix_data()
 
-    items_by_id = pretix_connector.items_by_id
+    item_names_by_id = pretix_connector.item_names_by_id
 
-    assert len(items_by_id) == 5
+    assert len(item_names_by_id) == 5
 
-    assert items_by_id[339041].names_by_locale["en"] == "Business"
-    assert items_by_id[163246].names_by_locale["en"] == "Conference"
-    assert items_by_id[163247].names_by_locale["en"] == "Tutorials"
-    assert items_by_id[339042].names_by_locale["en"] == "Personal"
-    assert items_by_id[163253].names_by_locale["en"] == "Combined (Conference + Tutorials)"
+    assert item_names_by_id[339041] == "Business"
+    assert item_names_by_id[163246] == "Conference"
+    assert item_names_by_id[163247] == "Tutorials"
+    assert item_names_by_id[339042] == "Personal"
+    assert item_names_by_id[163253] == "Combined (Conference + Tutorials)"
 
 
 @pytest.mark.asyncio
@@ -128,6 +128,25 @@ async def test_get_ticket(pretix_mock):
     ]
 
 
+async def test_cache(pretix_mock, tmp_path):
+    pretix_connector_1 = PretixConnector(
+        url=pretix_mock.base_url, token=PRETIX_API_TOKEN, cache_file=tmp_path / "pretix_cache.json"
+    )
+    assert not pretix_connector_1.item_names_by_id
+    assert not pretix_connector_1.tickets_by_key
+
+    # fetch data in connector 1
+    await pretix_connector_1.fetch_pretix_data()
+    assert pretix_connector_1.item_names_by_id
+    assert pretix_connector_1.tickets_by_key
+
+    pretix_connector_2 = PretixConnector(
+        url=pretix_mock.base_url, token=PRETIX_API_TOKEN, cache_file=tmp_path / "pretix_cache.json"
+    )
+    assert pretix_connector_1.item_names_by_id == pretix_connector_2.item_names_by_id
+    assert pretix_connector_1.tickets_by_key == pretix_connector_2.tickets_by_key
+
+
 async def test_get_ticket_handles_ticket_ids(pretix_mock):
     pretix_connector = PretixConnector(url=pretix_mock.base_url, token=PRETIX_API_TOKEN)
 
@@ -272,7 +291,9 @@ async def test_pagination(aiohttp_client, unused_tcp_port_factory):
     pretix_connector = PretixConnector(url=pretix_mock.base_url, token=PRETIX_API_TOKEN)
     await pretix_connector.fetch_pretix_data()
 
-    assert len(pretix_connector.items_by_id) == 5, "Only the first page of '/items' was fetched."
+    assert (
+        len(pretix_connector.item_names_by_id) == 5
+    ), "Only the first page of '/items' was fetched."
 
 
 @pytest.mark.asyncio

tests/registration/test_registration_logger.py

@@ -9,22 +9,26 @@
 def test_with_empty_file(tmp_path: Path) -> None:
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    assert not logger.is_registered(Ticket("ABC01", "John Doe", "Business", "Tutorials"))
+    assert not logger.is_registered(
+        Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials")
+    )
 
 
 def test_with_existing_file(tmp_path: Path) -> None:
     (tmp_path / "registrations.txt").write_text("ABC01-johndoe\n")
 
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    assert logger.is_registered(Ticket("ABC01", "John Doe", "Business", "Tutorials"))
+    assert logger.is_registered(
+        Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials")
+    )
 
 
 @pytest.mark.asyncio
 async def test_register_ticket_on_empty_log(tmp_path: Path) -> None:
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    ticket = Ticket("ABC01", "John Doe", "Business", "Tutorials")
+    ticket = Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials")
 
     await logger.mark_as_registered(ticket)
 
@@ -36,7 +40,7 @@ async def test_register_ticket_on_empty_log(tmp_path: Path) -> None:
 async def test_register_ticket_with_existing_file(tmp_path: Path) -> None:
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    ticket = Ticket("ABC01", "John Doe", "Business", "Tutorials")
+    ticket = Ticket(order="ABC01", name="John Doe", type="Business", variation="Tutorials")
 
     await logger.mark_as_registered(ticket)
 
@@ -50,7 +54,7 @@ async def test_register_ticket_with_existing_log(tmp_path: Path) -> None:
 
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    ticket = Ticket("ABC02", "Jane Doe", "Business", "Tutorials")
+    ticket = Ticket(order="ABC02", name="Jane Doe", type="Business", variation="Tutorials")
 
     await logger.mark_as_registered(ticket)
 
@@ -62,7 +66,7 @@ async def test_register_ticket_with_existing_log(tmp_path: Path) -> None:
 async def test_register_already_registered_ticket(tmp_path: Path) -> None:
     logger = RegistrationLogger(tmp_path / "registrations.txt")
 
-    ticket = Ticket("ABC02", "Jane Doe", "Business", "Tutorials")
+    ticket = Ticket(order="ABC02", name="Jane Doe", type="Business", variation="Tutorials")
 
     await logger.mark_as_registered(ticket)