Use newest PMAT (Metaculus fixes, supported_markets property)

prediction_market_agent/agents/invalid_agent/deploy.py

@@ -11,16 +11,14 @@
 
 
 class InvalidAgent(DeployableTraderAgent):
+    """This agent works only on Omen.
+    Because on Omen, after market is resolved as invalid, outcome tokens are worth equally, which means one can be profitable by buying the cheapest token.
+    Also the function to mark invalid questions is based on Omen-resolution rules."""
+
     bet_on_n_markets_per_run: int = 10
+    supported_markets = [MarketType.OMEN]
 
     def verify_market(self, market_type: MarketType, market: AgentMarket) -> bool:
-        if market_type != MarketType.OMEN:
-            raise RuntimeError(
-                "This agent works only on Omen."
-                " Because on Omen, after market is resolved as invalid, outcome tokens are worth equally, which means one can be profitable by buying the cheapest token."
-                " Also the function to mark invalid questions is based on Omen-resolution rules."
-            )
-
         if self.have_bet_on_market_since(market, since=self.same_market_bet_interval):
             return False
 

prediction_market_agent/agents/known_outcome_agent/deploy.py

@@ -20,6 +20,7 @@ class DeployableKnownOutcomeAgent(DeployableTraderAgent):
     model = "gpt-4-1106-preview"
     min_liquidity = 5
     bet_on_n_markets_per_run: int = 2
+    supported_markets = [MarketType.OMEN]
 
     def get_betting_strategy(self, market: AgentMarket) -> BettingStrategy:
         return KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.6)
@@ -28,10 +29,9 @@ def load(self) -> None:
         self.markets_with_known_outcomes: dict[str, Result] = {}
 
     def verify_market(self, market_type: MarketType, market: AgentMarket) -> bool:
-        if not isinstance(market, OmenAgentMarket):
-            raise NotImplementedError(
-                "This agent only supports predictions on Omen markets"
-            )
+        assert isinstance(
+            market, OmenAgentMarket
+        ), "It's true thanks to supported_markets property, this just makes mypy happy."
 
         # Assume very high probability markets are already known, and have
         # been correctly bet on, and therefore the value of betting on them

prediction_market_agent/agents/metaculus_agent/deploy.py

@@ -1,84 +1,82 @@
 import sys
 from typing import Sequence
 
-from prediction_market_agent_tooling.deploy.agent import DeployableAgent
+from prediction_market_agent_tooling.deploy.agent import DeployablePredictionAgent
 from prediction_market_agent_tooling.gtypes import Probability
 from prediction_market_agent_tooling.loggers import logger
-from prediction_market_agent_tooling.markets.agent_market import FilterBy, SortBy
+from prediction_market_agent_tooling.markets.agent_market import (
+    AgentMarket,
+    FilterBy,
+    SortBy,
+)
 from prediction_market_agent_tooling.markets.data_models import ProbabilisticAnswer
 from prediction_market_agent_tooling.markets.markets import MarketType
 from prediction_market_agent_tooling.markets.metaculus.metaculus import (
     MetaculusAgentMarket,
 )
-from prediction_market_agent_tooling.tools.utils import check_not_none
 
-from prediction_market_agent.agents.think_thoroughly_agent.think_thoroughly_agent import (
-    ThinkThoroughlyWithItsOwnResearch,
+from prediction_market_agent.agents.prophet_agent.deploy import (
+    DeployablePredictionProphetGPTo1PreviewAgent,
 )
-from prediction_market_agent.utils import DEFAULT_OPENAI_MODEL
 
 WARMUP_TOURNAMENT_ID = 3294
 TOURNAMENT_ID = 3349
 
 
-class DeployableMetaculusBotTournamentAgent(DeployableAgent):
-    model: str = DEFAULT_OPENAI_MODEL
+class DeployableMetaculusBotTournamentAgent(DeployablePredictionAgent):
+    bet_on_n_markets_per_run: int = (
+        sys.maxsize
+    )  # On Metaculus "betting" is free, we can just bet on everything available in one run.
     dummy_prediction: bool = False
     repeat_predictions: bool = False
     tournament_id: int = TOURNAMENT_ID
+    supported_markets = [MarketType.METACULUS]
 
-    def run(
-        self,
-        market_type: MarketType = MarketType.METACULUS,
-    ) -> None:
-        """
-        Submit predictions to Metaculus markets using the CrewAIAgentSubquestions
-
-        https://www.metaculus.com/notebooks/25525/announcing-the-ai-forecasting-benchmark-series--july-8-120k-in-prizes/
-        """
-
-        if market_type != MarketType.METACULUS:
-            raise ValueError("Only Metaculus markets are supported for this agent")
-
-        agent = ThinkThoroughlyWithItsOwnResearch(
-            model=self.model, enable_langfuse=self.enable_langfuse, memory=False
+    def load(self) -> None:
+        # Using this one because it had the lowest `p_yes mse` from the `match_bets_with_langfuse_traces.py` evaluation at the time of writing this.
+        self.agent = DeployablePredictionProphetGPTo1PreviewAgent(
+            enable_langfuse=self.enable_langfuse
         )
+
+    def get_markets(self, market_type: MarketType) -> Sequence[AgentMarket]:  # type: ignore # TODO: Needs to be decided in https://github.com/gnosis/prediction-market-agent/pull/511#discussion_r1810034688 and then I'll implement it here.
         markets: Sequence[
             MetaculusAgentMarket
         ] = MetaculusAgentMarket.get_binary_markets(
-            limit=sys.maxsize,
+            limit=self.bet_on_n_markets_per_run,
             tournament_id=self.tournament_id,
             filter_by=FilterBy.OPEN,
             sort_by=SortBy.NEWEST,
         )
-        logger.info(f"Found {len(markets)} open markets to submit predictions for.")
+        return markets
 
-        if not self.repeat_predictions:
-            # Filter out markets that we have already answered
-            markets = [market for market in markets if not market.have_predicted]
-            logger.info(
-                f"Found {len(markets)} unanswered markets to submit predictions for."
-            )
+    def verify_market(self, market_type: MarketType, market: AgentMarket) -> bool:
+        assert isinstance(
+            market, MetaculusAgentMarket
+        ), "Just making mypy happy. It's true thanks to the check in the `run` method via `supported_markets`."
 
-        for market in markets:
-            logger.info(f"Answering market {market.id}, question: {market.question}")
-            if not self.dummy_prediction:
-                # TODO incorporate 'Resolution criteria', 'Fine print', and
-                # 'Background info' into the prompt given to the agent.
-                answer = agent.answer_binary_market(
-                    market.question, created_time=market.created_time
-                )
-            else:
-                answer = ProbabilisticAnswer(
-                    p_yes=Probability(0.5),
-                    reasoning="Just a test.",
-                    confidence=0.5,
-                )
+        # Filter out the market if the agent isn't configured to re-bet.
+        if not self.repeat_predictions and market.have_predicted:
+            return False
 
-            if answer is None:
-                logger.error("No answer was given. Skipping")
-            else:
-                market.submit_prediction(
-                    p_yes=answer.p_yes,
-                    reasoning=check_not_none(answer.reasoning),
-                )
+        # Otherwise all markets on Metaculus are fine.
+        return True
+
+    def answer_binary_market(self, market: AgentMarket) -> ProbabilisticAnswer | None:
+        assert isinstance(
+            market, MetaculusAgentMarket
+        ), "Just making mypy happy. It's true thanks to the check in the `run` method via `supported_markets`."
+        logger.info(f"Answering market {market.id}, question: {market.question}")
+        answer: ProbabilisticAnswer | None
+        if not self.dummy_prediction:
+            full_question = f"""Question: {market.question}
+Question's description: {market.description}
+Question's fine print: {market.fine_print} 
+Question's resolution criteria: {market.resolution_criteria}"""
+            answer = self.agent.agent.predict(full_question).outcome_prediction
+        else:
+            answer = ProbabilisticAnswer(
+                p_yes=Probability(0.5),
+                reasoning="Just a test.",
+                confidence=0.5,
+            )
+        return answer

prediction_market_agent/agents/specialized_agent/deploy.py

@@ -30,6 +30,7 @@ class GetMarketCreatorsStalkerMarkets:
     bet_on_n_markets_per_run = MAX_AVAILABLE_MARKETS
     # These tends to be long-running markets, it's not interesting to bet on them too much.
     same_market_bet_interval = timedelta(days=7)
+    supported_markets: t.Sequence[MarketType] = [MarketType.OMEN]
 
     def get_markets(
         self,
@@ -38,11 +39,6 @@ def get_markets(
         sort_by: SortBy = SortBy.CLOSING_SOONEST,
         filter_by: FilterBy = FilterBy.OPEN,
     ) -> t.Sequence[OmenAgentMarket]:
-        if market_type != MarketType.OMEN:
-            raise RuntimeError(
-                f"{self.__class__.__name__} agent can only be used on Omen markets."
-            )
-
         available_markets = [
             OmenAgentMarket.from_data_model(m)
             for m in OmenSubgraphHandler().get_omen_binary_markets_simple(

prediction_market_agent/agents/think_thoroughly_agent/deploy.py

@@ -51,7 +51,9 @@ def get_betting_strategy(self, market: AgentMarket) -> BettingStrategy:
 
 
 if __name__ == "__main__":
-    agent = DeployableThinkThoroughlyAgent(place_bet=False)
+    agent = DeployableThinkThoroughlyAgent(
+        place_trades=False, store_prediction=False, store_trades=False
+    )
     agent.deploy_local(
         market_type=MarketType.OMEN,
         sleep_time=540,

pyproject.toml

@@ -35,12 +35,11 @@ poetry = "^1.7.1"
 poetry-plugin-export = "^1.6.0"
 functions-framework = "^3.5.0"
 cron-validator = "^1.0.8"
-prediction-market-agent-tooling = { version = "^0.54.0", extras = ["langchain", "google"] }
+prediction-market-agent-tooling = { git = "https://github.com/gnosis/prediction-market-agent-tooling.git", rev = "3101ea542185c16befc0464f674eac854b40a700", extras = ["langchain", "google"] }
 pydantic-settings = "^2.1.0"
 autoflake = "^2.2.1"
 isort = "^5.13.2"
 markdownify = "^0.11.6"
-tavily-python = "^0.3.9"
 microchain-python = { git = "https://github.com/galatolofederico/microchain.git", rev = "de75f1d4a073b7c54f824b409733f4f70d40a61b" }
 pysqlite3-binary = {version="^0.5.2.post3", markers = "sys_platform == 'linux'"}
 psycopg2-binary = "^2.9.9"
@@ -52,7 +51,7 @@ tweepy = "^4.14.0"
 langchain-pinecone = "^0.1.1"
 pinatapy-vourhey = "^0.2.0"
 plotly = "^5.22.0"
-prediction-prophet = { git = "https://github.com/agentcoinorg/predictionprophet.git", rev = "69b3ab619b25eab5890e58d5dc65be3aadf5bb65" }
+prediction-prophet = { git = "https://github.com/agentcoinorg/predictionprophet.git", rev = "693bfef2f9392a9af443433626f43cfeedaf1466" }
 transformers = "^4.43.3"
 openfactverification-kongzii = "^0.2.0"
 

scripts/agent_app.py

@@ -78,7 +78,9 @@ def predict(
     enable_langfuse: bool,
 ) -> None:
     agent = AgentClass(
-        place_bet=False,
+        place_trades=False,
+        store_prediction=False,
+        store_trades=False,
         enable_langfuse=enable_langfuse,
     )