From 32202f8b4b979aa70337df97e7ec29c42e5affdd Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 22 Feb 2024 09:56:36 +0100 Subject: [PATCH 1/3] Add number of markets and resolution stats to the benchmark output --- .../benchmark/benchmark.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/prediction_market_agent_tooling/benchmark/benchmark.py b/prediction_market_agent_tooling/benchmark/benchmark.py index c349fc4a..1e3ba528 100644 --- a/prediction_market_agent_tooling/benchmark/benchmark.py +++ b/prediction_market_agent_tooling/benchmark/benchmark.py @@ -412,6 +412,30 @@ def get_markets_summary(self) -> t.Dict[str, t.List[str | float]]: ] return markets_summary + def get_markets_overall_info(self) -> dict[str, list[str | float]]: + return { + "Number of markets": [len(self.markets)], + "Proportion resolved": [ + sum(1 for m in self.markets if m.is_resolved) / len(self.markets) + ], + "Proportion YES": [ + sum( + 1 + for m in self.markets + if m.probable_resolution == MarketResolution.YES + ) + / len(self.markets) + ], + "Proportion NO": [ + sum( + 1 + for m in self.markets + if m.probable_resolution == MarketResolution.NO + ) + / len(self.markets) + ], + } + def calculate_expected_returns( self, prediction: Prediction, market: Market ) -> float | None: @@ -498,6 +522,9 @@ def compute_expected_returns_summary( def generate_markdown_report(self) -> str: md = "# Comparison Report\n\n" + md += "## Overall Info\n\n" + md += pd.DataFrame(self.get_markets_overall_info()).to_markdown(index=False) + md += "\n\n" md += "## Summary Statistics\n\n" md += pd.DataFrame(self.compute_metrics()).to_markdown(index=False) md += "\n\n" From 7467b0635264715a7d36bc48a526e1b132a8829d Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 22 Feb 2024 14:38:22 +0100 Subject: [PATCH 2/3] update generate_markdown_report structure --- prediction_market_agent_tooling/benchmark/benchmark.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/prediction_market_agent_tooling/benchmark/benchmark.py b/prediction_market_agent_tooling/benchmark/benchmark.py index 1e3ba528..1b553c2d 100644 --- a/prediction_market_agent_tooling/benchmark/benchmark.py +++ b/prediction_market_agent_tooling/benchmark/benchmark.py @@ -522,16 +522,17 @@ def compute_expected_returns_summary( def generate_markdown_report(self) -> str: md = "# Comparison Report\n\n" - md += "## Overall Info\n\n" + md += "## Market Results\n\n" md += pd.DataFrame(self.get_markets_overall_info()).to_markdown(index=False) md += "\n\n" - md += "## Summary Statistics\n\n" + md += "## Agent Results\n\n" + md += "### Summary Statistics\n\n" md += pd.DataFrame(self.compute_metrics()).to_markdown(index=False) md += "\n\n" - md += "## Markets\n\n" + md += "### Markets\n\n" md += pd.DataFrame(self.get_markets_summary()).to_markdown(index=False) md += "\n\n" - md += "## Expected value\n\n" + md += "### Expected value\n\n" overall_summary, per_market = self.compute_expected_returns_summary() md += pd.DataFrame(overall_summary).to_markdown(index=False) md += "\n\n" From ff2289b47bed3d0bf67ea8530a48425faefeebf8 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 22 Feb 2024 14:39:00 +0100 Subject: [PATCH 3/3] update func name too --- prediction_market_agent_tooling/benchmark/benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prediction_market_agent_tooling/benchmark/benchmark.py b/prediction_market_agent_tooling/benchmark/benchmark.py index 1b553c2d..4bff73ac 100644 --- a/prediction_market_agent_tooling/benchmark/benchmark.py +++ b/prediction_market_agent_tooling/benchmark/benchmark.py @@ -412,7 +412,7 @@ def get_markets_summary(self) -> t.Dict[str, t.List[str | float]]: ] return markets_summary - def get_markets_overall_info(self) -> dict[str, list[str | float]]: + def get_markets_results(self) -> dict[str, list[str | float]]: return { "Number of markets": [len(self.markets)], "Proportion resolved": [ @@ -523,7 +523,7 @@ def compute_expected_returns_summary( def generate_markdown_report(self) -> str: md = "# Comparison Report\n\n" md += "## Market Results\n\n" - md += pd.DataFrame(self.get_markets_overall_info()).to_markdown(index=False) + md += pd.DataFrame(self.get_markets_results()).to_markdown(index=False) md += "\n\n" md += "## Agent Results\n\n" md += "### Summary Statistics\n\n"