Monitor market resolutions (#16)

.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.env.monitor
 .venv
 env/
 venv/

examples/monitor/monitor.py

@@ -1,21 +1,8 @@
-from datetime import datetime, timedelta
-from zoneinfo import ZoneInfo
-
 import streamlit as st
 
-from prediction_market_agent_tooling.markets.manifold import get_authenticated_user
-from prediction_market_agent_tooling.monitor.markets.manifold import (
-    DeployedManifoldAgent,
-)
-from prediction_market_agent_tooling.monitor.monitor import monitor_agent
+from prediction_market_agent_tooling.monitor.monitor_app import monitor_app
 
 if __name__ == "__main__":
-    start_time = datetime.now() - timedelta(weeks=1)
-    agent = DeployedManifoldAgent(
-        name="foo",
-        start_time=start_time.astimezone(ZoneInfo("UTC")),
-        manifold_user_id=get_authenticated_user().id,
-    )
     st.set_page_config(layout="wide")  # Best viewed with a wide screen
-    st.title(f"Monitoring Agent: '{agent.name}'")
-    monitor_agent(agent)
+    st.title(f"Monitoring")
+    monitor_app()

prediction_market_agent_tooling/benchmark/utils.py

@@ -1,10 +1,14 @@
 import json
 import typing as t
+from datetime import datetime
 from enum import Enum
 
+import pytz
 import requests
 from pydantic import BaseModel, validator
 
+MANIFOLD_API_LIMIT = 1000  # Manifold will only return up to 1000 markets
+
 
 class EvaluatedQuestion(BaseModel):
     question: str
@@ -23,6 +27,7 @@ class Market(BaseModel):
     p_yes: float
     volume: float
     is_resolved: bool
+    created_time: datetime
     resolution: str | None = None
     outcomePrices: list[float] | None = None
 
@@ -34,6 +39,12 @@ def _validate_outcome_prices(cls, value: list[float] | None) -> list[float] | No
             raise ValueError("outcomePrices must have exactly 2 elements.")
         return value
 
+    @validator("created_time")
+    def _validate_created_time(cls, value: datetime) -> datetime:
+        if value.tzinfo is None:
+            value = value.replace(tzinfo=pytz.UTC)
+        return value
+
     @property
     def p_no(self) -> float:
         return 1 - self.p_yes
@@ -103,22 +114,24 @@ def save(self, path: str) -> None:
     @staticmethod
     def load(path: str) -> "PredictionsCache":
         with open(path, "r") as f:
-            return PredictionsCache.parse_obj(json.load(f))
+            return PredictionsCache.model_validate(json.load(f))
 
 
 def get_manifold_markets(
-    number: int = 100,
-    excluded_questions: t.List[str] = [],
+    limit: int = 100,
+    offset: int = 0,
     filter_: t.Literal[
         "open", "closed", "resolved", "closing-this-month", "closing-next-month"
     ] = "open",
+    sort: t.Literal["liquidity", "score", "newest"] = "liquidity",
 ) -> t.List[Market]:
     url = "https://api.manifold.markets/v0/search-markets"
     params = {
         "term": "",
-        "sort": "liquidity",
+        "sort": sort,
         "filter": filter_,
-        "limit": f"{number + len(excluded_questions)}",
+        "limit": f"{limit}",
+        "offset": offset,
         "contractType": "BINARY",  # TODO support CATEGORICAL markets
     }
     response = requests.get(url, params=params)
@@ -132,27 +145,85 @@ def get_manifold_markets(
     fields_map = {
         "probability": "p_yes",
         "isResolved": "is_resolved",
+        "createdTime": "created_time",
     }
 
     def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]:
         return {mapping.get(k, k): v for k, v in old.items()}
 
-    markets = [Market.parse_obj(_map_fields(m, fields_map)) for m in markets_json]
+    markets = [Market.model_validate(_map_fields(m, fields_map)) for m in markets_json]
+
+    return markets
+
+
+def get_manifold_markets_paged(
+    number: int = 100,
+    filter_: t.Literal[
+        "open", "closed", "resolved", "closing-this-month", "closing-next-month"
+    ] = "open",
+    sort: t.Literal["liquidity", "score", "newest"] = "liquidity",
+    starting_offset: int = 0,
+    excluded_questions: set[str] | None = None,
+) -> t.List[Market]:
+    markets: list[Market] = []
+
+    offset = starting_offset
+    while len(markets) < number:
+        new_markets = get_manifold_markets(
+            limit=min(MANIFOLD_API_LIMIT, number - len(markets)),
+            offset=offset,
+            filter_=filter_,
+            sort=sort,
+        )
+        if not new_markets:
+            break
+        markets.extend(
+            market
+            for market in new_markets
+            if not excluded_questions or market.question not in excluded_questions
+        )
+        offset += len(new_markets)
+
+    return markets
 
-    # Filter out markets with excluded questions
-    markets = [m for m in markets if m.question not in excluded_questions]
 
-    return markets[:number]
+def get_manifold_markets_dated(
+    oldest_date: datetime,
+    filter_: t.Literal[
+        "open", "closed", "resolved", "closing-this-month", "closing-next-month"
+    ] = "open",
+    excluded_questions: set[str] | None = None,
+) -> t.List[Market]:
+    markets: list[Market] = []
+
+    offset = 0
+    while True:
+        new_markets = get_manifold_markets(
+            limit=MANIFOLD_API_LIMIT,
+            offset=offset,
+            filter_=filter_,
+            sort="newest",  # Enforce sorting by newest, because there aren't date filters on the API.
+        )
+        if not new_markets:
+            break
+        for market in new_markets:
+            if market.created_time < oldest_date:
+                return markets
+            if not excluded_questions or market.question not in excluded_questions:
+                markets.append(market)
+            offset += 1
+
+    return markets
 
 
 def get_polymarket_markets(
-    number: int = 100,
-    excluded_questions: t.List[str] = [],
+    limit: int = 100,
     active: bool | None = True,
     closed: bool | None = False,
+    excluded_questions: set[str] | None = None,
 ) -> t.List[Market]:
     params: dict[str, str | int] = {
-        "_limit": number + len(excluded_questions),
+        "_limit": limit,
     }
     if active is not None:
         params["active"] = "true" if active else "false"
@@ -167,8 +238,7 @@ def get_polymarket_markets(
         if m_json["outcomes"] != ["Yes", "No"]:
             continue
 
-        if m_json["question"] in excluded_questions:
-            print(f"Skipping market with 'excluded question': {m_json['question']}")
+        if excluded_questions and m_json["question"] in excluded_questions:
             continue
 
         markets.append(
@@ -178,6 +248,7 @@ def get_polymarket_markets(
                 p_yes=m_json["outcomePrices"][
                     0
                 ],  # For binary markets on Polymarket, the first outcome is "Yes" and outcomePrices are equal to probabilities.
+                created_time=m_json["created_at"],
                 outcomePrices=m_json["outcomePrices"],
                 volume=m_json["volume"],
                 is_resolved=False,
@@ -190,15 +261,15 @@ def get_polymarket_markets(
 def get_markets(
     number: int,
     source: MarketSource,
-    excluded_questions: t.List[str] = [],
+    excluded_questions: set[str] | None = None,
 ) -> t.List[Market]:
     if source == MarketSource.MANIFOLD:
-        return get_manifold_markets(
+        return get_manifold_markets_paged(
             number=number, excluded_questions=excluded_questions
         )
     elif source == MarketSource.POLYMARKET:
         return get_polymarket_markets(
-            number=number, excluded_questions=excluded_questions
+            limit=number, excluded_questions=excluded_questions
         )
     else:
         raise ValueError(f"Unknown market source: {source}")

prediction_market_agent_tooling/markets/manifold.py

@@ -81,10 +81,10 @@ def place_bet(amount: Mana, market_id: str, outcome: bool) -> None:
         )
 
 
-def get_authenticated_user() -> ManifoldUser:
+def get_authenticated_user(api_key: str) -> ManifoldUser:
     url = "https://api.manifold.markets/v0/me"
     headers = {
-        "Authorization": f"Key {APIKeys().manifold_api_key}",
+        "Authorization": f"Key {api_key}",
         "Content-Type": "application/json",
     }
     response = requests.get(url, headers=headers)

prediction_market_agent_tooling/monitor/monitor.py

@@ -1,12 +1,16 @@
 import typing as t
 from datetime import datetime
+from itertools import groupby
 
 import altair as alt
+import numpy as np
 import pandas as pd
 import streamlit as st
 from pydantic import BaseModel
 
+from prediction_market_agent_tooling.benchmark.utils import Market
 from prediction_market_agent_tooling.markets.data_models import ResolvedBet
+from prediction_market_agent_tooling.tools.utils import should_not_happen
 
 
 class DeployedAgent(BaseModel):
@@ -20,6 +24,9 @@ def get_resolved_bets(self) -> list[ResolvedBet]:
 
 def monitor_agent(agent: DeployedAgent) -> None:
     agent_bets = agent.get_resolved_bets()
+    if not agent_bets:
+        st.warning(f"No resolved bets found for {agent.name}.")
+        return
     bets_info = {
         "Market Question": [bet.market_question for bet in agent_bets],
         "Bet Amount": [bet.amount.amount for bet in agent_bets],
@@ -47,7 +54,6 @@ def monitor_agent(agent: DeployedAgent) -> None:
         profit_df.groupby("Date")["Cumulative Profit"].sum().cumsum().reset_index()
     )
     profit_df["Cumulative Profit"] = profit_df["Cumulative Profit"].astype(float)
-    st.empty()
     st.altair_chart(
         alt.Chart(profit_df)
         .mark_line()
@@ -60,6 +66,80 @@ def monitor_agent(agent: DeployedAgent) -> None:
     )
 
     # Table of resolved bets
-    st.empty()
     st.subheader("Resolved Bet History")
     st.table(bets_df)
+
+
+def monitor_market(open_markets: list[Market], resolved_markets: list[Market]) -> None:
+    date_to_open_yes_proportion = {
+        d: np.mean([int(m.p_yes > 0.5) for m in markets])
+        for d, markets in groupby(open_markets, lambda x: x.created_time.date())
+    }
+    date_to_resolved_yes_proportion = {
+        d: np.mean(
+            [
+                (
+                    1
+                    if m.resolution == "YES"
+                    else (
+                        0
+                        if m.resolution == "NO"
+                        else should_not_happen(f"Unexpected resolution: {m.resolution}")
+                    )
+                )
+                for m in markets
+            ]
+        )
+        for d, markets in groupby(resolved_markets, lambda x: x.created_time.date())
+    }
+
+    df_open = pd.DataFrame(
+        date_to_open_yes_proportion.items(), columns=["date", "open_proportion"]
+    )
+    df_open["open_label"] = "Open's yes proportion"
+    df_resolved = pd.DataFrame(
+        date_to_resolved_yes_proportion.items(), columns=["date", "resolved_proportion"]
+    )
+    df_resolved["resolved_label"] = "Resolved's yes proportion"
+
+    df = pd.merge(df_open, df_resolved, on="date")
+
+    open_chart = (
+        alt.Chart(df)
+        .mark_line()
+        .encode(x="date:T", y="open_proportion:Q", color="open_label:N")
+    )
+
+    resolved_chart = (
+        alt.Chart(df)
+        .mark_line()
+        .encode(x="date:T", y="resolved_proportion:Q", color="resolved_label:N")
+    )
+
+    st.altair_chart(
+        alt.layer(open_chart, resolved_chart).interactive(),  # type: ignore # Doesn't expect `LayerChart`, but `Chart`, yet it works.
+        use_container_width=True,
+    )
+
+    all_open_markets_yes_mean = np.mean([int(m.p_yes > 0.5) for m in open_markets])
+    all_resolved_markets_yes_mean = np.mean(
+        [
+            (
+                1
+                if m.resolution == "YES"
+                else (
+                    0
+                    if m.resolution == "NO"
+                    else should_not_happen(f"Unexpected resolution: {m.resolution}")
+                )
+            )
+            for m in resolved_markets
+        ]
+    )
+    st.markdown(
+        f"Total number of open markets {len(open_markets)} and resolved markets {len(resolved_markets)}"
+        "\n\n"
+        f"Mean proportion of 'YES' in open markets: {all_open_markets_yes_mean:.2f}"
+        "\n\n"
+        f"Mean proportion of 'YES' in resolved markets: {all_resolved_markets_yes_mean:.2f}"
+    )