From b692c58891f8e7ba213c2bbfcf5801f4566893fd Mon Sep 17 00:00:00 2001
From: John Yang <byjohnyang@gmail.com>
Date: Sat, 15 Jun 2024 17:35:36 -0400
Subject: [PATCH] Add notes to leaderboard

---
 index.html             | 10 +++++++++-
 submit.html            |  4 +---
 template/template.html | 10 +++++++++-
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/index.html b/index.html
index 8d8df9a..0e7415a 100644
--- a/index.html
+++ b/index.html
@@ -405,8 +405,16 @@ <h2 class="text-title">Leaderboard</h2>
               </table>
             </div>
             <p class="text-content">
-              The <span style="color:#0ea7ff;"><b>% Resolved</b></span> metric refers to the percentage of SWE-bench instances (2294 total)
+              - The <span style="color:#0ea7ff;"><b>% Resolved</b></span> metric refers to the percentage of SWE-bench instances (2294 total)
               that were <i>resolved</i> by the model.
+              <br>
+              - The leaderboard will be updated once a week on Monday.
+              <br>
+              - If you would like to submit your model to the leaderboard, please check the <a href="submit.html">submission</a> page.
+              <br>
+              - All submissions are Pass@1, do not use
+              <code style="color:black;background-color:#ddd;border-radius: 0.25em">hints_text</code>,
+              and are in the unassisted setting.
             </p>
           </div>
         </div>
diff --git a/submit.html b/submit.html
index 44718a8..740a769 100644
--- a/submit.html
+++ b/submit.html
@@ -133,10 +133,8 @@ <h3>
                 <ol>
                     <li>The use of the <code>hints_text</code> field is <i>not</i> allowed. See our explanation <a href="https://github.com/princeton-nlp/SWE-bench/issues/133">here</a>.</li>
                     <li>The result should be pass@1. There should be one execution log per task instance for all 2294 task instances.</li>
+                    <li>The result should <i>not</i> be in the "Oracle" retrieval setting. The agent cannot be told the correct files to edit, where "correct" refers to the files modified by the reference solution patch.</li>
                 </ol>
-                <p>
-                    We do not enforce the guidelines 
-                </p>
             </div>
         </div>
         <div class="content-wrapper">
diff --git a/template/template.html b/template/template.html
index d02789a..af2b332 100644
--- a/template/template.html
+++ b/template/template.html
@@ -159,8 +159,16 @@ <h2 class="text-title">Leaderboard</h2>
               </table>
             </div>
             <p class="text-content">
-              The <span style="color:#0ea7ff;"><b>% Resolved</b></span> metric refers to the percentage of SWE-bench instances (2294 total)
+              - The <span style="color:#0ea7ff;"><b>% Resolved</b></span> metric refers to the percentage of SWE-bench instances (2294 total)
               that were <i>resolved</i> by the model.
+              <br>
+              - The leaderboard will be updated once a week on Monday.
+              <br>
+              - If you would like to submit your model to the leaderboard, please check the <a href="submit.html">submission</a> page.
+              <br>
+              - All submissions are Pass@1, do not use
+              <code style="color:black;background-color:#ddd;border-radius: 0.25em">hints_text</code>,
+              and are in the unassisted setting.
             </p>
           </div>
         </div>