From f4801c27f40de8a40e41a32cea0fc4aa7664e140 Mon Sep 17 00:00:00 2001
From: brunneis <dev@brunneis.com>
Date: Thu, 14 Dec 2023 16:49:50 +0000
Subject: [PATCH] Improve judge prompt

---
 src/lib/prompts.py | 50 ++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/src/lib/prompts.py b/src/lib/prompts.py
index 6a2400a..c9b2dff 100644
--- a/src/lib/prompts.py
+++ b/src/lib/prompts.py
@@ -27,31 +27,27 @@
 )
 
 judge_predictions_prompt = PromptTemplate.from_template(
-    """Your role is to to judge a code audit:
-- blockchain: ethereum
-- language: solidity
-
-You are provided with:
-- <ground_truth>: the real and only vulnerabilities that exist in the code
-- <audit>: the audit to be judged by an external system
-
-Dos
-- Analyze the <audit> based on the real vulnerabilities listed on the <ground_truth>
-- Detect true positives
-- Detect false positives
-- Detect false negatives
-
-Dont's
-- Vulnerabilities not listed in the <ground_truth> are considered false positives
-- Judge based on your knowledge how well the <audit> is comparing it to the <ground_truth>
-- True positives lists vulnerabilities other than the ones listed in the <ground_truth>
-
-The output should be **exclusively** in json format with keys:
-- false_negatives: array of issue categories
-- false_positives: array of issue categories
-- true_positives: array of issue categories
-
-Where each element of the array is the category name of the vulnerability.
+    """<date>Wednesday 15th of January, 2030</date>
+<role>
+    - As an expert Solidity security auditor, your expertise lies in identifying and addressing vulnerabilities in smart contracts designed for the Ethereum Virtual Machine (EVM).
+    - Solidity, being the primary language for these contracts, has its unique set of potential security pitfalls. 
+</role>
+
+<tasks>
+    - Judge the <audit> based on the real vulnerabilities listed on the <ground_truth>
+    - Detect true positives: those that are listed in the <ground_truth> and are also listed in the <audit>.
+    - Detect false positives: those that are not listed in the <ground_truth> but are listed in the <audit>.
+    - Detect false negatives: those that are listed in the <ground_truth> but are not listed in the <audit>.
+</tasks>
+
+<important>
+This is the most important instruction: the output **MUST** be exclusively a JSON in the following format:
+{
+    "true_positives": [ground_truth_category: str],
+    "false_positives": [ground_truth_category: str],
+    "false_negatives": [ground_truth_category: str]
+}
+</important>
 
 <ground_truth>
 {key}
@@ -59,5 +55,7 @@
 
 <audit>
 {prediction}
-</audit>"""
+</audit>
+
+Thank you for your help, I will tip you 500 USD if you do it fine."""
 )