From 9075b90008dc1354e21189ffdbf80713653214a2 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Thu, 24 Oct 2024 04:25:37 +0100
Subject: [PATCH 01/20] feat: dynamic ground truths

---
 src/handlers/find-ground-truths.ts | 86 ++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 src/handlers/find-ground-truths.ts

diff --git a/src/handlers/find-ground-truths.ts b/src/handlers/find-ground-truths.ts
new file mode 100644
index 0000000..c861463
--- /dev/null
+++ b/src/handlers/find-ground-truths.ts
@@ -0,0 +1,86 @@
+import OpenAI from "openai";
+import { Context } from "../types";
+import { logger } from "../helpers/errors";
+
+const FIND_GROUND_TRUTHS_SYSTEM_MESSAGE = `Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
+    These ground truths are high-level abstractions that encapsulate the key aspects of the task.
+    They serve to guide and inform our code review model's interpretation of the task by providing clear, concise, and explicit insights.
+    
+    Each ground truth should:
+    - Be succinct and easy to understand.
+    - Directly pertain to the task at hand.
+    - Focus on essential requirements, behaviors, or assumptions involved in the task.
+
+    Example:
+    Task: Implement a function that adds two numbers.
+    Ground Truths:
+    - The function should accept two numerical inputs.
+    - The function should return the sum of the two inputs.
+    - Inputs must be validated to ensure they are numbers.
+    
+    Based on the given task, generate similar ground truths adhering to a maximum of 10.
+    
+    Return a JSON parsable array of strings representing the ground truths, without comment or directive.`;
+
+function validateGroundTruths(truthsString: string): string[] {
+  let truths;
+  try {
+    truths = JSON.parse(truthsString);
+  } catch (err) {
+    throw logger.error("Failed to parse ground truths");
+  }
+  if (!Array.isArray(truths)) {
+    throw logger.error("Ground truths must be an array");
+  }
+
+  if (truths.length > 10) {
+    throw logger.error("Ground truths must not exceed 10");
+  }
+
+  truths.forEach((truth: string) => {
+    if (typeof truth !== "string") {
+      throw logger.error("Each ground truth must be a string");
+    }
+  });
+
+  return truths;
+}
+
+export async function findGroundTruths(context: Context, groundTruthSource: string) {
+  const {
+    env: { OPENAI_API_KEY },
+    config: { openAiBaseUrl, model },
+  } = context;
+
+  const openAi = new OpenAI({
+    apiKey: OPENAI_API_KEY,
+    ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
+  });
+
+  const res = await openAi.chat.completions.create({
+    messages: [
+      {
+        role: "system",
+        content: FIND_GROUND_TRUTHS_SYSTEM_MESSAGE,
+      },
+      {
+        role: "user",
+        content: groundTruthSource,
+      },
+    ],
+    /**
+     * I've used the config model here but in my opinion,
+     * we should optimize this for a quicker response which
+     * means no advanced reasoning models. rfc
+     */
+    model: model,
+  });
+
+  const output = res.choices[0].message.content;
+
+  if (!output) {
+    throw logger.error("Failed to produce a ground truths response");
+  }
+
+  return validateGroundTruths(output);
+}

From 78dbb463cfa08d082b912e8731aa7024299a8442 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:12:16 +0100
Subject: [PATCH 02/20] chore: bump TS to latest, gitignore, cspell

---
 .cspell.json | 4 +++-
 .gitignore   | 1 +
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.cspell.json b/.cspell.json
index bbe91d8..31d9f87 100644
--- a/.cspell.json
+++ b/.cspell.json
@@ -30,7 +30,9 @@
     "mixtral",
     "nemo",
     "Reranking",
-    "mistralai"
+    "mistralai",
+    "Typeguard",
+    "typeguards"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],
diff --git a/.gitignore b/.gitignore
index 12274bf..3dcd5ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ cypress/screenshots
 script.ts
 .wrangler
 test-dashboard.md
+t.ts
\ No newline at end of file
diff --git a/package.json b/package.json
index 94d1357..8cf5000 100644
--- a/package.json
+++ b/package.json
@@ -66,7 +66,7 @@
     "prettier": "3.3.2",
     "ts-jest": "29.1.5",
     "tsx": "4.15.6",
-    "typescript": "5.4.5",
+    "typescript": "^5.6.3",
     "typescript-eslint": "7.13.1",
     "wrangler": "^3.81.0"
   },
diff --git a/yarn.lock b/yarn.lock
index 8dccaee..bb77c02 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6468,10 +6468,10 @@ typescript-eslint@7.13.1:
     "@typescript-eslint/parser" "7.13.1"
     "@typescript-eslint/utils" "7.13.1"
 
-typescript@5.4.5:
-  version "5.4.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611"
-  integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==
+typescript@^5.6.3:
+  version "5.6.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b"
+  integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==
 
 ufo@^1.5.4:
   version "1.5.4"

From 63685b2a7f03115569c6ae8f094680fbb917e0ee Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:12:56 +0100
Subject: [PATCH 03/20] chore: update ask-llm with dynamic truths

---
 src/handlers/ask-llm.ts | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/handlers/ask-llm.ts b/src/handlers/ask-llm.ts
index 30112c3..a121392 100644
--- a/src/handlers/ask-llm.ts
+++ b/src/handlers/ask-llm.ts
@@ -5,6 +5,8 @@ import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues
 import { recursivelyFetchLinkedIssues } from "../helpers/issue-fetching";
 import { formatChatHistory } from "../helpers/format-chat-history";
 import { optimizeContext } from "../helpers/issue";
+import { fetchRepoDependencies, fetchRepoLanguageStats } from "./ground-truths/chat-bot";
+import { findGroundTruths } from "./ground-truths/find-ground-truths";
 
 /**
  * Asks a question to GPT and returns the response
@@ -62,12 +64,17 @@ export async function askGpt(context: Context, question: string, formattedChat:
   // const reRankedChat = formattedChat.length > 0 ? await context.adapters.voyage.reranker.reRankResults(formattedChat.filter(text => text !== ""), question, 300) : [];
   similarText = similarText.filter((text) => text !== "");
   const rerankedText = similarText.length > 0 ? await context.adapters.voyage.reranker.reRankResults(similarText, question) : [];
-  return context.adapters.openai.completions.createCompletion(
-    question,
-    model,
-    rerankedText,
-    formattedChat,
-    ["typescript", "github", "cloudflare worker", "actions", "jest", "supabase", "openai"],
-    UBIQUITY_OS_APP_NAME
-  );
+
+  const languages = await fetchRepoLanguageStats(context);
+  const { dependencies, devDependencies } = await fetchRepoDependencies(context);
+  const groundTruths = await findGroundTruths(context, "chat-bot", {
+    languages,
+    dependencies,
+    devDependencies,
+  });
+
+  console.log("languages: ", languages);
+  console.log("Ground Truths: ", groundTruths);
+
+  return context.adapters.openai.completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME);
 }

From c764b11b5f2e21318281ff876df0fed0851e9755 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:20:34 +0100
Subject: [PATCH 04/20] chore: Logs export

---
 src/helpers/errors.ts | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 src/helpers/errors.ts

diff --git a/src/helpers/errors.ts b/src/helpers/errors.ts
new file mode 100644
index 0000000..354a399
--- /dev/null
+++ b/src/helpers/errors.ts
@@ -0,0 +1,3 @@
+import { Logs } from "@ubiquity-dao/ubiquibot-logger"; // import is fixed in #13
+
+export const logger = new Logs("debug");

From b6e2489055e9910d8634e800ab6e40357d64755f Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:21:37 +0100
Subject: [PATCH 05/20] chore: groundTruths completion util

---
 .../create-ground-truth-completion.ts         | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 src/handlers/ground-truths/create-ground-truth-completion.ts

diff --git a/src/handlers/ground-truths/create-ground-truth-completion.ts b/src/handlers/ground-truths/create-ground-truth-completion.ts
new file mode 100644
index 0000000..512a975
--- /dev/null
+++ b/src/handlers/ground-truths/create-ground-truth-completion.ts
@@ -0,0 +1,36 @@
+import OpenAI from "openai";
+import { Context } from "../../types";
+import { CompletionsModelHelper, ModelApplications } from "../../types/llm";
+
+export async function createGroundTruthCompletion<TApp extends ModelApplications>(
+  context: Context,
+  groundTruthSource: string,
+  systemMsg: string,
+  model: CompletionsModelHelper<TApp>
+): Promise<string | null> {
+  const {
+    env: { OPENAI_API_KEY },
+    config: { openAiBaseUrl },
+  } = context;
+
+  const openAi = new OpenAI({
+    apiKey: OPENAI_API_KEY,
+    ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
+  });
+
+  const res = await openAi.chat.completions.create({
+    messages: [
+      {
+        role: "system",
+        content: systemMsg,
+      },
+      {
+        role: "user",
+        content: groundTruthSource,
+      },
+    ],
+    model: model,
+  });
+
+  return res.choices[0].message.content;
+}

From 22b1cba0f30151ae4f122757b61a829d7d6dc3f5 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:22:41 +0100
Subject: [PATCH 06/20] chore: groundTruths system msg util

---
 .../ground-truths/create-system-message.ts      | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 src/handlers/ground-truths/create-system-message.ts

diff --git a/src/handlers/ground-truths/create-system-message.ts b/src/handlers/ground-truths/create-system-message.ts
new file mode 100644
index 0000000..3e839ad
--- /dev/null
+++ b/src/handlers/ground-truths/create-system-message.ts
@@ -0,0 +1,17 @@
+export function createGroundTruthSysMsg({ truthRules, example, conditions }: { truthRules: string[]; example: string[]; conditions?: string[] }) {
+  return `
+Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
+These ground truths are high-level abstractions that encapsulate the tech stack and dependencies of the repository.
+  
+Each ground truth should:
+- ${truthRules.join("\n- ")}
+  
+Example:
+${example.join("\n")}
+  
+${conditions ? `Conditions:\n${conditions}` : ""}
+  
+Generate similar ground truths adhering to a maximum of 10.
+  
+Return a JSON parsable array of strings representing the ground truths, without comment or directive.`;
+}

From b5bbb2b0175494c79c4f4294830a1451fde40317 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:23:23 +0100
Subject: [PATCH 07/20] feat: fetch repo langs and deps

---
 src/handlers/ground-truths/chat-bot.ts | 64 ++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 src/handlers/ground-truths/chat-bot.ts

diff --git a/src/handlers/ground-truths/chat-bot.ts b/src/handlers/ground-truths/chat-bot.ts
new file mode 100644
index 0000000..30072ef
--- /dev/null
+++ b/src/handlers/ground-truths/chat-bot.ts
@@ -0,0 +1,64 @@
+import { Context } from "../../types";
+import { logger } from "../../helpers/errors";
+
+export async function fetchRepoDependencies(context: Context) {
+  const {
+    octokit,
+    payload: {
+      repository: {
+        owner: { login: owner },
+        name: repo,
+      },
+    },
+  } = context;
+
+  const { data: packageJson } = await octokit.repos.getContent({
+    owner,
+    repo,
+    path: "package.json",
+  });
+
+  if ("content" in packageJson) {
+    return extractDependencies(JSON.parse(Buffer.from(packageJson.content, "base64").toString()));
+  } else {
+    throw logger.error(`No package.json found in ${owner}/${repo}`);
+  }
+}
+
+export function extractDependencies(packageJson: Record<string, Record<string, string>>) {
+  const { dependencies, devDependencies } = packageJson;
+
+  return {
+    dependencies,
+    devDependencies,
+  };
+}
+
+export async function fetchRepoLanguageStats(context: Context) {
+  const {
+    octokit,
+    payload: {
+      repository: {
+        owner: { login: owner },
+        name: repo,
+      },
+    },
+  } = context;
+
+  const { data: languages } = await octokit.repos.listLanguages({
+    owner,
+    repo,
+  });
+
+  const totalBytes = Object.values(languages).reduce((acc, bytes) => acc + bytes, 0);
+
+  const stats = Object.entries(languages).reduce(
+    (acc, [language, bytes]) => {
+      acc[language] = bytes / totalBytes;
+      return acc;
+    },
+    {} as Record<string, number>
+  );
+
+  return Array.from(Object.entries(stats)).sort((a, b) => b[1] - a[1]);
+}

From 0567c5cd0dc7ff2019aaa32ec1447f26cd560ec2 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:23:56 +0100
Subject: [PATCH 08/20] feat: findGroundTruths

---
 src/handlers/find-ground-truths.ts            | 86 -------------------
 .../ground-truths/find-ground-truths.ts       | 36 ++++++++
 2 files changed, 36 insertions(+), 86 deletions(-)
 delete mode 100644 src/handlers/find-ground-truths.ts
 create mode 100644 src/handlers/ground-truths/find-ground-truths.ts

diff --git a/src/handlers/find-ground-truths.ts b/src/handlers/find-ground-truths.ts
deleted file mode 100644
index c861463..0000000
--- a/src/handlers/find-ground-truths.ts
+++ /dev/null
@@ -1,86 +0,0 @@
-import OpenAI from "openai";
-import { Context } from "../types";
-import { logger } from "../helpers/errors";
-
-const FIND_GROUND_TRUTHS_SYSTEM_MESSAGE = `Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
-    These ground truths are high-level abstractions that encapsulate the key aspects of the task.
-    They serve to guide and inform our code review model's interpretation of the task by providing clear, concise, and explicit insights.
-    
-    Each ground truth should:
-    - Be succinct and easy to understand.
-    - Directly pertain to the task at hand.
-    - Focus on essential requirements, behaviors, or assumptions involved in the task.
-
-    Example:
-    Task: Implement a function that adds two numbers.
-    Ground Truths:
-    - The function should accept two numerical inputs.
-    - The function should return the sum of the two inputs.
-    - Inputs must be validated to ensure they are numbers.
-    
-    Based on the given task, generate similar ground truths adhering to a maximum of 10.
-    
-    Return a JSON parsable array of strings representing the ground truths, without comment or directive.`;
-
-function validateGroundTruths(truthsString: string): string[] {
-  let truths;
-  try {
-    truths = JSON.parse(truthsString);
-  } catch (err) {
-    throw logger.error("Failed to parse ground truths");
-  }
-  if (!Array.isArray(truths)) {
-    throw logger.error("Ground truths must be an array");
-  }
-
-  if (truths.length > 10) {
-    throw logger.error("Ground truths must not exceed 10");
-  }
-
-  truths.forEach((truth: string) => {
-    if (typeof truth !== "string") {
-      throw logger.error("Each ground truth must be a string");
-    }
-  });
-
-  return truths;
-}
-
-export async function findGroundTruths(context: Context, groundTruthSource: string) {
-  const {
-    env: { OPENAI_API_KEY },
-    config: { openAiBaseUrl, model },
-  } = context;
-
-  const openAi = new OpenAI({
-    apiKey: OPENAI_API_KEY,
-    ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
-  });
-
-  const res = await openAi.chat.completions.create({
-    messages: [
-      {
-        role: "system",
-        content: FIND_GROUND_TRUTHS_SYSTEM_MESSAGE,
-      },
-      {
-        role: "user",
-        content: groundTruthSource,
-      },
-    ],
-    /**
-     * I've used the config model here but in my opinion,
-     * we should optimize this for a quicker response which
-     * means no advanced reasoning models. rfc
-     */
-    model: model,
-  });
-
-  const output = res.choices[0].message.content;
-
-  if (!output) {
-    throw logger.error("Failed to produce a ground truths response");
-  }
-
-  return validateGroundTruths(output);
-}
diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
new file mode 100644
index 0000000..accdf64
--- /dev/null
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -0,0 +1,36 @@
+import { Context } from "../../types";
+import { AppParamsHelper, ModelApplications } from "../../types/llm";
+import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "./prompts";
+import { chatBotPayloadTypeguard, codeReviewPayloadTypeguard } from "../../types/typeguards";
+import { validateGroundTruths } from "./validate";
+import { logger } from "../../helpers/errors";
+import { createGroundTruthCompletion } from "./create-ground-truth-completion";
+import { createGroundTruthSysMsg } from "./create-system-message";
+
+export async function findGroundTruths<TApp extends ModelApplications = ModelApplications>(
+  context: Context,
+  application: TApp,
+  params: AppParamsHelper<TApp>
+): Promise<string[]> {
+  const systemMsgObj = GROUND_TRUTHS_SYSTEM_MESSAGES[application];
+  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
+
+  if (chatBotPayloadTypeguard(params)) {
+    return findChatBotTruths(context, params, systemMsg);
+  } else if (codeReviewPayloadTypeguard(params)) {
+    return findCodeReviewTruths(context, params, systemMsg);
+  } else {
+    throw logger.error("Invalid payload type for ground truths");
+  }
+}
+
+async function findChatBotTruths(context: Context, params: AppParamsHelper<"chat-bot">, systemMsg: string): Promise<string[]> {
+  const truths = await createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
+  return validateGroundTruths(truths);
+}
+
+async function findCodeReviewTruths(context: Context, params: AppParamsHelper<"code-review">, systemMsg: string): Promise<string[]> {
+  const { taskSpecification } = params;
+  const truths = await createGroundTruthCompletion<"code-review">(context, taskSpecification, systemMsg, "gpt-4o");
+  return validateGroundTruths(truths);
+}

From 5e1547225e5f325f573dc9dd6eaea208a3de4501 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:25:14 +0100
Subject: [PATCH 09/20] chore: improve templates, type helper utils

---
 src/handlers/ground-truths/prompts.ts | 68 +++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 src/handlers/ground-truths/prompts.ts

diff --git a/src/handlers/ground-truths/prompts.ts b/src/handlers/ground-truths/prompts.ts
new file mode 100644
index 0000000..bd3b99f
--- /dev/null
+++ b/src/handlers/ground-truths/prompts.ts
@@ -0,0 +1,68 @@
+import { ModelApplications } from "../../types/llm";
+
+export type GroundTruthsSystemMessage<TApp extends ModelApplications = ModelApplications> = TApp extends "code-review"
+  ? typeof CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE
+  : TApp extends "chat-bot"
+    ? typeof CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE
+    : never;
+
+export type GroundTruthsSystemMessageTemplate = {
+  truthRules: string[];
+  example: string[];
+  conditions?: string[];
+};
+
+const CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE = {
+  example: [
+    `Using the input provided, your goal is to produce an array of strings that represent "Ground Truths."
+        These ground truths are high-level abstractions that encapsulate the key aspects of the task.
+        They serve to guide and inform our code review model's interpretation of the task by providing clear, concise, and explicit insights.
+        
+        Each ground truth should:
+        - Be succinct and easy to understand.
+        - Directly pertain to the task at hand.
+        - Focus on essential requirements, behaviors, or assumptions involved in the task.
+    
+        Example:
+        Task: Implement a function that adds two numbers.
+        Ground Truths:
+        - The function should accept two numerical inputs.
+        - The function should return the sum of the two inputs.
+        - Inputs must be validated to ensure they are numbers.
+        
+        Based on the given task, generate similar ground truths adhering to a maximum of 10.
+        
+        Return a JSON parsable array of strings representing the ground truths, without comment or directive.`,
+  ],
+  truthRules: [],
+  conditions: [],
+};
+
+const CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE = {
+  truthRules: [
+    "Be succinct and easy to understand.",
+    "Use only the information provided in the input.",
+    "Focus on essential requirements, behaviors, or assumptions involved in the repository.",
+  ],
+  example: [
+    "Languages: { TypeScript: 60%, JavaScript: 15%, HTML: 10%, CSS: 5%, ... }",
+    "Dependencies: Esbuild, Wrangler, React, Tailwind CSS, ms, React-carousel, React-icons, ...",
+    "Dev Dependencies: @types/node, @types/jest, @mswjs, @testing-library/react, @testing-library/jest-dom, @Cypress ...",
+    "Ground Truths:",
+    "- The repo predominantly uses TypeScript, with JavaScript, HTML, and CSS also present.",
+    "- The repo is a React project that uses Tailwind CSS.",
+    "- The project is built with Esbuild and deployed with Wrangler, indicating a Cloudflare Workers project.",
+    "- The repo tests use Jest, Cypress, mswjs, and React Testing Library.",
+  ],
+  conditions: [
+    "Assume your output builds the foundation for a chatbot to understand the repository when asked an arbitrary query.",
+    "Do not list every language or dependency, focus on the most prevalent ones.",
+    "Focus on what is essential to understand the repository at a high level.",
+    "Brevity is key.",
+  ],
+};
+
+export const GROUND_TRUTHS_SYSTEM_MESSAGES: Record<ModelApplications, GroundTruthsSystemMessageTemplate> = {
+  "code-review": CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE,
+  "chat-bot": CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE,
+} as const;

From 23e62dbf964cfc08d03ce65f480d5cc3aefc84b6 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:27:05 +0100
Subject: [PATCH 10/20] chore: rename .d.ts, consolidate types

---
 src/handlers/ground-truths/prompts.ts | 14 +------
 src/types/llm.d.ts                    | 19 ----------
 src/types/llm.ts                      | 53 +++++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 32 deletions(-)
 delete mode 100644 src/types/llm.d.ts
 create mode 100644 src/types/llm.ts

diff --git a/src/handlers/ground-truths/prompts.ts b/src/handlers/ground-truths/prompts.ts
index bd3b99f..e6159cb 100644
--- a/src/handlers/ground-truths/prompts.ts
+++ b/src/handlers/ground-truths/prompts.ts
@@ -1,16 +1,4 @@
-import { ModelApplications } from "../../types/llm";
-
-export type GroundTruthsSystemMessage<TApp extends ModelApplications = ModelApplications> = TApp extends "code-review"
-  ? typeof CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE
-  : TApp extends "chat-bot"
-    ? typeof CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE
-    : never;
-
-export type GroundTruthsSystemMessageTemplate = {
-  truthRules: string[];
-  example: string[];
-  conditions?: string[];
-};
+import { GroundTruthsSystemMessageTemplate, ModelApplications } from "../../types/llm";
 
 const CODE_REVIEW_GROUND_TRUTHS_SYSTEM_MESSAGE = {
   example: [
diff --git a/src/types/llm.d.ts b/src/types/llm.d.ts
deleted file mode 100644
index 5bfaa19..0000000
--- a/src/types/llm.d.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-export type StreamlinedComment = {
-  id: number;
-  user?: string;
-  body?: string;
-  org: string;
-  repo: string;
-  issueUrl: string;
-  specOrBody?: {
-    html: string;
-    text: string;
-  };
-};
-
-export type StreamlinedComments = {
-  issueNumber: number;
-  repo: string;
-  org: string;
-  comments: StreamlinedComment[];
-};
diff --git a/src/types/llm.ts b/src/types/llm.ts
new file mode 100644
index 0000000..a20a016
--- /dev/null
+++ b/src/types/llm.ts
@@ -0,0 +1,53 @@
+import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "../handlers/ground-truths/prompts";
+
+export type ModelApplications = "code-review" | "chat-bot";
+
+type ChatBotAppParams = {
+  languages: [string, number][];
+  dependencies: Record<string, string>;
+  devDependencies: Record<string, string>;
+};
+
+type CodeReviewAppParams = {
+  taskSpecification: string;
+};
+
+export type AppParamsHelper<TApp extends ModelApplications> = TApp extends "code-review"
+  ? CodeReviewAppParams
+  : TApp extends "chat-bot"
+  ? ChatBotAppParams
+  : never;
+
+export type CompletionsModelHelper<TApp extends ModelApplications> = TApp extends "code-review" ? "gpt-4o" : TApp extends "chat-bot" ? "o1-mini" : never;
+
+export type GroundTruthsSystemMessage<TApp extends ModelApplications = ModelApplications> = TApp extends "code-review"
+  ? typeof GROUND_TRUTHS_SYSTEM_MESSAGES["code-review"]
+  : TApp extends "chat-bot"
+  ? typeof GROUND_TRUTHS_SYSTEM_MESSAGES["chat-bot"]
+  : never;
+
+export type GroundTruthsSystemMessageTemplate = {
+  truthRules: string[];
+  example: string[];
+  conditions?: string[];
+};
+
+export type StreamlinedComment = {
+  id: number;
+  user?: string;
+  body?: string;
+  org: string;
+  repo: string;
+  issueUrl: string;
+  specOrBody?: {
+    html: string;
+    text: string;
+  };
+};
+
+export type StreamlinedComments = {
+  issueNumber: number;
+  repo: string;
+  org: string;
+  comments: StreamlinedComment[];
+};

From b4ff6bc2be7a1f8bc668a22e383f10e15a20612d Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:27:30 +0100
Subject: [PATCH 11/20] chore: validate util, typeguards

---
 src/handlers/ground-truths/validate.ts | 28 ++++++++++++++++++++++++++
 src/types/typeguards.ts                |  9 +++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 src/handlers/ground-truths/validate.ts
 create mode 100644 src/types/typeguards.ts

diff --git a/src/handlers/ground-truths/validate.ts b/src/handlers/ground-truths/validate.ts
new file mode 100644
index 0000000..de90955
--- /dev/null
+++ b/src/handlers/ground-truths/validate.ts
@@ -0,0 +1,28 @@
+import { logger } from "../../helpers/errors";
+
+export function validateGroundTruths(truthsString: string | null): string[] {
+  let truths;
+  if (!truthsString) {
+    throw logger.error("Failed to generate ground truths");
+  }
+  try {
+    truths = JSON.parse(truthsString);
+  } catch (err) {
+    throw logger.error("Failed to parse ground truths");
+  }
+  if (!Array.isArray(truths)) {
+    throw logger.error("Ground truths must be an array");
+  }
+
+  if (truths.length > 10) {
+    throw logger.error("Ground truths must not exceed 10");
+  }
+
+  truths.forEach((truth: string) => {
+    if (typeof truth !== "string") {
+      throw logger.error("Each ground truth must be a string");
+    }
+  });
+
+  return truths;
+}
diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts
new file mode 100644
index 0000000..f965c90
--- /dev/null
+++ b/src/types/typeguards.ts
@@ -0,0 +1,9 @@
+import { AppParamsHelper } from "./llm";
+
+export function chatBotPayloadTypeguard(payload: unknown): payload is AppParamsHelper<"chat-bot"> {
+  return typeof payload === "object" && payload !== null && "repoLanguages" in payload && "repoDependencies" in payload && "chatBotPrompt" in payload;
+}
+
+export function codeReviewPayloadTypeguard(payload: unknown): payload is AppParamsHelper<"code-review"> {
+  return typeof payload === "object" && payload !== null && "taskSpecification" in payload && "codeReviewModelPrompt" in payload;
+}

From a145b346f7bc326c3e9b057565f450e33719c652 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:31:25 +0100
Subject: [PATCH 12/20] chore: use default template worker port

---
 .github/workflows/compute.yml              | 10 +++++-----
 .github/workflows/update-configuration.yml |  2 +-
 package.json                               |  4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml
index 285665e..533c5ec 100644
--- a/.github/workflows/compute.yml
+++ b/.github/workflows/compute.yml
@@ -45,8 +45,8 @@ jobs:
         run: yarn tsx ./src/main.ts
         id: command-ask
         env:
-            SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-            SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-            VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-            UBIQUITY_OS_APP_NAME: ${{ secrets.UBIQUITY_OS_APP_NAME }}
\ No newline at end of file
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+          VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          UBIQUITY_OS_APP_NAME: ${{ secrets.UBIQUITY_OS_APP_NAME }}
diff --git a/.github/workflows/update-configuration.yml b/.github/workflows/update-configuration.yml
index 2d366d6..b92a487 100644
--- a/.github/workflows/update-configuration.yml
+++ b/.github/workflows/update-configuration.yml
@@ -18,4 +18,4 @@ jobs:
           commitMessage: "chore: updated manifest.json and dist build"
           nodeVersion: "20.10.0"
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/package.json b/package.json
index 8cf5000..4e5888f 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,7 @@
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
     "prepare": "husky install",
     "test": "jest --setupFiles dotenv/config --coverage",
-    "worker": "wrangler dev --env dev --port 5000"
+    "worker": "wrangler dev --env dev --port 4000"
   },
   "keywords": [
     "typescript",
@@ -84,4 +84,4 @@
       "@commitlint/config-conventional"
     ]
   }
-}
+}
\ No newline at end of file

From 49c9dcdeb232d93c1fa577c1e890af542e4df9da Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 18:34:33 +0100
Subject: [PATCH 13/20] chore: remove logs, formatting, prompt tweak

---
 package.json                                  |  2 +-
 src/handlers/ask-llm.ts                       |  3 ---
 .../create-ground-truth-completion.ts         | 22 ++++++++++---------
 .../ground-truths/create-system-message.ts    |  2 +-
 .../ground-truths/find-ground-truths.ts       |  6 +++--
 src/handlers/ground-truths/prompts.ts         |  3 ++-
 src/handlers/ground-truths/validate.ts        |  1 +
 src/types/llm.ts                              | 10 ++++-----
 src/types/typeguards.ts                       |  2 +-
 9 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/package.json b/package.json
index 4e5888f..71bee04 100644
--- a/package.json
+++ b/package.json
@@ -84,4 +84,4 @@
       "@commitlint/config-conventional"
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/src/handlers/ask-llm.ts b/src/handlers/ask-llm.ts
index a121392..7f3a7b3 100644
--- a/src/handlers/ask-llm.ts
+++ b/src/handlers/ask-llm.ts
@@ -73,8 +73,5 @@ export async function askGpt(context: Context, question: string, formattedChat:
     devDependencies,
   });
 
-  console.log("languages: ", languages);
-  console.log("Ground Truths: ", groundTruths);
-
   return context.adapters.openai.completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME);
 }
diff --git a/src/handlers/ground-truths/create-ground-truth-completion.ts b/src/handlers/ground-truths/create-ground-truth-completion.ts
index 512a975..e1d8a49 100644
--- a/src/handlers/ground-truths/create-ground-truth-completion.ts
+++ b/src/handlers/ground-truths/create-ground-truth-completion.ts
@@ -18,17 +18,19 @@ export async function createGroundTruthCompletion<TApp extends ModelApplications
     ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
   });
 
+  const msgs = [
+    {
+      role: "system",
+      content: systemMsg,
+    },
+    {
+      role: "user",
+      content: groundTruthSource,
+    },
+  ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+
   const res = await openAi.chat.completions.create({
-    messages: [
-      {
-        role: "system",
-        content: systemMsg,
-      },
-      {
-        role: "user",
-        content: groundTruthSource,
-      },
-    ],
+    messages: msgs,
     model: model,
   });
 
diff --git a/src/handlers/ground-truths/create-system-message.ts b/src/handlers/ground-truths/create-system-message.ts
index 3e839ad..2c29c78 100644
--- a/src/handlers/ground-truths/create-system-message.ts
+++ b/src/handlers/ground-truths/create-system-message.ts
@@ -9,7 +9,7 @@ Each ground truth should:
 Example:
 ${example.join("\n")}
   
-${conditions ? `Conditions:\n${conditions}` : ""}
+${conditions ? `Conditions:\n${conditions.join("\n")}` : ""}
   
 Generate similar ground truths adhering to a maximum of 10.
   
diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
index accdf64..44d244e 100644
--- a/src/handlers/ground-truths/find-ground-truths.ts
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -16,9 +16,11 @@ export async function findGroundTruths<TApp extends ModelApplications = ModelApp
   const systemMsg = createGroundTruthSysMsg(systemMsgObj);
 
   if (chatBotPayloadTypeguard(params)) {
-    return findChatBotTruths(context, params, systemMsg);
+    const { dependencies, devDependencies, languages } = params;
+    return findChatBotTruths(context, { dependencies, devDependencies, languages }, systemMsg);
   } else if (codeReviewPayloadTypeguard(params)) {
-    return findCodeReviewTruths(context, params, systemMsg);
+    const { taskSpecification } = params;
+    return findCodeReviewTruths(context, { taskSpecification }, systemMsg);
   } else {
     throw logger.error("Invalid payload type for ground truths");
   }
diff --git a/src/handlers/ground-truths/prompts.ts b/src/handlers/ground-truths/prompts.ts
index e6159cb..d0258e1 100644
--- a/src/handlers/ground-truths/prompts.ts
+++ b/src/handlers/ground-truths/prompts.ts
@@ -46,7 +46,8 @@ const CHAT_BOT_GROUND_TRUTHS_SYSTEM_MESSAGE = {
     "Assume your output builds the foundation for a chatbot to understand the repository when asked an arbitrary query.",
     "Do not list every language or dependency, focus on the most prevalent ones.",
     "Focus on what is essential to understand the repository at a high level.",
-    "Brevity is key.",
+    "Brevity is key. Use zero formatting. Do not wrap in quotes, backticks, or other characters.",
+    `response === ["some", "array", "of", "strings"]`,
   ],
 };
 
diff --git a/src/handlers/ground-truths/validate.ts b/src/handlers/ground-truths/validate.ts
index de90955..2068b95 100644
--- a/src/handlers/ground-truths/validate.ts
+++ b/src/handlers/ground-truths/validate.ts
@@ -5,6 +5,7 @@ export function validateGroundTruths(truthsString: string | null): string[] {
   if (!truthsString) {
     throw logger.error("Failed to generate ground truths");
   }
+
   try {
     truths = JSON.parse(truthsString);
   } catch (err) {
diff --git a/src/types/llm.ts b/src/types/llm.ts
index a20a016..f01a70d 100644
--- a/src/types/llm.ts
+++ b/src/types/llm.ts
@@ -15,16 +15,16 @@ type CodeReviewAppParams = {
 export type AppParamsHelper<TApp extends ModelApplications> = TApp extends "code-review"
   ? CodeReviewAppParams
   : TApp extends "chat-bot"
-  ? ChatBotAppParams
-  : never;
+    ? ChatBotAppParams
+    : never;
 
 export type CompletionsModelHelper<TApp extends ModelApplications> = TApp extends "code-review" ? "gpt-4o" : TApp extends "chat-bot" ? "o1-mini" : never;
 
 export type GroundTruthsSystemMessage<TApp extends ModelApplications = ModelApplications> = TApp extends "code-review"
-  ? typeof GROUND_TRUTHS_SYSTEM_MESSAGES["code-review"]
+  ? (typeof GROUND_TRUTHS_SYSTEM_MESSAGES)["code-review"]
   : TApp extends "chat-bot"
-  ? typeof GROUND_TRUTHS_SYSTEM_MESSAGES["chat-bot"]
-  : never;
+    ? (typeof GROUND_TRUTHS_SYSTEM_MESSAGES)["chat-bot"]
+    : never;
 
 export type GroundTruthsSystemMessageTemplate = {
   truthRules: string[];
diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts
index f965c90..c582d42 100644
--- a/src/types/typeguards.ts
+++ b/src/types/typeguards.ts
@@ -1,7 +1,7 @@
 import { AppParamsHelper } from "./llm";
 
 export function chatBotPayloadTypeguard(payload: unknown): payload is AppParamsHelper<"chat-bot"> {
-  return typeof payload === "object" && payload !== null && "repoLanguages" in payload && "repoDependencies" in payload && "chatBotPrompt" in payload;
+  return typeof payload === "object" && payload !== null && "languages" in payload && "dependencies" in payload;
 }
 
 export function codeReviewPayloadTypeguard(payload: unknown): payload is AppParamsHelper<"code-review"> {

From 96f98af66aae4fdbe68a5287a04ca01934ec8df4 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 19:47:38 +0100
Subject: [PATCH 14/20] chore: test handlers - add readme fetch, stats - fix
 openai resp

---
 tests/__mocks__/handlers.ts | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/__mocks__/handlers.ts b/tests/__mocks__/handlers.ts
index 20503d9..be7ba62 100644
--- a/tests/__mocks__/handlers.ts
+++ b/tests/__mocks__/handlers.ts
@@ -7,7 +7,7 @@ import issueTemplate from "./issue-template";
  */
 export const handlers = [
   http.post("https://api.openai.com/v1/chat/completions", () => {
-    const answer = `This is a mock answer for the chat`;
+    const answer = `${JSON.stringify(["This is a mock response from OpenAI"])}`;
 
     return HttpResponse.json({
       usage: {
@@ -85,4 +85,16 @@ export const handlers = [
       db.pull.findFirst({ where: { owner: { equals: owner as string }, repo: { equals: repo as string }, number: { equals: Number(pullNumber) } } })
     )
   ),
+  http.get("https://api.github.com/repos/:owner/:repo/languages", ({ params: { owner, repo } }) =>
+    HttpResponse.json(db.repo.findFirst({ where: { owner: { login: { equals: owner as string } }, name: { equals: repo as string } } }))
+  ),
+  http.get("https://api.github.com/repos/:owner/:repo/contents/:path", () =>
+    HttpResponse.json({
+      type: "file",
+      encoding: "base64",
+      size: 5362,
+      name: "README.md",
+      content: Buffer.from(JSON.stringify({ content: "This is a mock README file" })).toString("base64"),
+    })
+  ),
 ];

From ebd3c41bd99ddb85062876ed26498387039b83ff Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 19:58:44 +0100
Subject: [PATCH 15/20] chore: remove obj decon

---
 src/handlers/ground-truths/find-ground-truths.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
index 44d244e..045cae2 100644
--- a/src/handlers/ground-truths/find-ground-truths.ts
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -32,7 +32,6 @@ async function findChatBotTruths(context: Context, params: AppParamsHelper<"chat
 }
 
 async function findCodeReviewTruths(context: Context, params: AppParamsHelper<"code-review">, systemMsg: string): Promise<string[]> {
-  const { taskSpecification } = params;
-  const truths = await createGroundTruthCompletion<"code-review">(context, taskSpecification, systemMsg, "gpt-4o");
+  const truths = await createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
   return validateGroundTruths(truths);
 }

From 359451e2341cc7f5d1a9b13b12f7ae2327107f01 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 20:09:34 +0100
Subject: [PATCH 16/20] chore: knip fix

---
 .../ground-truths/find-ground-truths.ts       | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
index 045cae2..23fd92b 100644
--- a/src/handlers/ground-truths/find-ground-truths.ts
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -1,5 +1,5 @@
 import { Context } from "../../types";
-import { AppParamsHelper, ModelApplications } from "../../types/llm";
+import { AppParamsHelper, GroundTruthsSystemMessage, ModelApplications } from "../../types/llm";
 import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "./prompts";
 import { chatBotPayloadTypeguard, codeReviewPayloadTypeguard } from "../../types/typeguards";
 import { validateGroundTruths } from "./validate";
@@ -13,25 +13,36 @@ export async function findGroundTruths<TApp extends ModelApplications = ModelApp
   params: AppParamsHelper<TApp>
 ): Promise<string[]> {
   const systemMsgObj = GROUND_TRUTHS_SYSTEM_MESSAGES[application];
-  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
+
+  // params are deconstructed to show quickly what's being passed to the function
 
   if (chatBotPayloadTypeguard(params)) {
     const { dependencies, devDependencies, languages } = params;
-    return findChatBotTruths(context, { dependencies, devDependencies, languages }, systemMsg);
+    return findChatBotTruths(context, { dependencies, devDependencies, languages }, systemMsgObj);
   } else if (codeReviewPayloadTypeguard(params)) {
     const { taskSpecification } = params;
-    return findCodeReviewTruths(context, { taskSpecification }, systemMsg);
+    return findCodeReviewTruths(context, { taskSpecification }, systemMsgObj);
   } else {
     throw logger.error("Invalid payload type for ground truths");
   }
 }
 
-async function findChatBotTruths(context: Context, params: AppParamsHelper<"chat-bot">, systemMsg: string): Promise<string[]> {
+async function findChatBotTruths(
+  context: Context,
+  params: AppParamsHelper<"chat-bot">,
+  systemMsgObj: GroundTruthsSystemMessage<"chat-bot">
+): Promise<string[]> {
+  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
   const truths = await createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
   return validateGroundTruths(truths);
 }
 
-async function findCodeReviewTruths(context: Context, params: AppParamsHelper<"code-review">, systemMsg: string): Promise<string[]> {
+async function findCodeReviewTruths(
+  context: Context,
+  params: AppParamsHelper<"code-review">,
+  systemMsgObj: GroundTruthsSystemMessage<"code-review">
+): Promise<string[]> {
+  const systemMsg = createGroundTruthSysMsg(systemMsgObj);
   const truths = await createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
   return validateGroundTruths(truths);
 }

From f7fa7b5b70d779622c61f64d3a2f0e689491a775 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 20:35:25 +0100
Subject: [PATCH 17/20] chore: move groundTruthCompletion into adapters

---
 src/adapters/openai/helpers/completions.ts    | 36 ++++++++++++++++++
 .../create-ground-truth-completion.ts         | 38 -------------------
 .../ground-truths/find-ground-truths.ts       |  7 ++--
 3 files changed, 40 insertions(+), 41 deletions(-)
 delete mode 100644 src/handlers/ground-truths/create-ground-truth-completion.ts

diff --git a/src/adapters/openai/helpers/completions.ts b/src/adapters/openai/helpers/completions.ts
index f68f305..cd15059 100644
--- a/src/adapters/openai/helpers/completions.ts
+++ b/src/adapters/openai/helpers/completions.ts
@@ -1,6 +1,7 @@
 import OpenAI from "openai";
 import { Context } from "../../../types";
 import { SuperOpenAi } from "./openai";
+import { CompletionsModelHelper, ModelApplications } from "../../../types/llm";
 const MAX_TOKENS = 7000;
 
 export interface CompletionsType {
@@ -76,4 +77,39 @@ export class Completions extends SuperOpenAi {
     }
     return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 } };
   }
+
+  async createGroundTruthCompletion<TApp extends ModelApplications>(
+    context: Context,
+    groundTruthSource: string,
+    systemMsg: string,
+    model: CompletionsModelHelper<TApp>
+  ): Promise<string | null> {
+    const {
+      env: { OPENAI_API_KEY },
+      config: { openAiBaseUrl },
+    } = context;
+
+    const openAi = new OpenAI({
+      apiKey: OPENAI_API_KEY,
+      ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
+    });
+
+    const msgs = [
+      {
+        role: "system",
+        content: systemMsg,
+      },
+      {
+        role: "user",
+        content: groundTruthSource,
+      },
+    ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+
+    const res = await openAi.chat.completions.create({
+      messages: msgs,
+      model: model,
+    });
+
+    return res.choices[0].message.content;
+  }
 }
diff --git a/src/handlers/ground-truths/create-ground-truth-completion.ts b/src/handlers/ground-truths/create-ground-truth-completion.ts
deleted file mode 100644
index e1d8a49..0000000
--- a/src/handlers/ground-truths/create-ground-truth-completion.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-import OpenAI from "openai";
-import { Context } from "../../types";
-import { CompletionsModelHelper, ModelApplications } from "../../types/llm";
-
-export async function createGroundTruthCompletion<TApp extends ModelApplications>(
-  context: Context,
-  groundTruthSource: string,
-  systemMsg: string,
-  model: CompletionsModelHelper<TApp>
-): Promise<string | null> {
-  const {
-    env: { OPENAI_API_KEY },
-    config: { openAiBaseUrl },
-  } = context;
-
-  const openAi = new OpenAI({
-    apiKey: OPENAI_API_KEY,
-    ...(openAiBaseUrl && { baseURL: openAiBaseUrl }),
-  });
-
-  const msgs = [
-    {
-      role: "system",
-      content: systemMsg,
-    },
-    {
-      role: "user",
-      content: groundTruthSource,
-    },
-  ] as OpenAI.Chat.Completions.ChatCompletionMessageParam[];
-
-  const res = await openAi.chat.completions.create({
-    messages: msgs,
-    model: model,
-  });
-
-  return res.choices[0].message.content;
-}
diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
index 23fd92b..6ed67cb 100644
--- a/src/handlers/ground-truths/find-ground-truths.ts
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -4,7 +4,6 @@ import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "./prompts";
 import { chatBotPayloadTypeguard, codeReviewPayloadTypeguard } from "../../types/typeguards";
 import { validateGroundTruths } from "./validate";
 import { logger } from "../../helpers/errors";
-import { createGroundTruthCompletion } from "./create-ground-truth-completion";
 import { createGroundTruthSysMsg } from "./create-system-message";
 
 export async function findGroundTruths<TApp extends ModelApplications = ModelApplications>(
@@ -32,8 +31,9 @@ async function findChatBotTruths(
   params: AppParamsHelper<"chat-bot">,
   systemMsgObj: GroundTruthsSystemMessage<"chat-bot">
 ): Promise<string[]> {
+  const { adapters: { openai: { completions } } } = context;
   const systemMsg = createGroundTruthSysMsg(systemMsgObj);
-  const truths = await createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
+  const truths = await completions.createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
   return validateGroundTruths(truths);
 }
 
@@ -42,7 +42,8 @@ async function findCodeReviewTruths(
   params: AppParamsHelper<"code-review">,
   systemMsgObj: GroundTruthsSystemMessage<"code-review">
 ): Promise<string[]> {
+  const { adapters: { openai: { completions } } } = context;
   const systemMsg = createGroundTruthSysMsg(systemMsgObj);
-  const truths = await createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
+  const truths = await completions.createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
   return validateGroundTruths(truths);
 }

From 5351bae4934ef468170492cc5ef4ee2e3fbbcc9f Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 22:06:30 +0100
Subject: [PATCH 18/20] chore: mock ground truths completion fn

---
 src/handlers/ground-truths/find-ground-truths.ts | 12 ++++++++++--
 tests/main.test.ts                               |  3 +++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/handlers/ground-truths/find-ground-truths.ts b/src/handlers/ground-truths/find-ground-truths.ts
index 6ed67cb..d441710 100644
--- a/src/handlers/ground-truths/find-ground-truths.ts
+++ b/src/handlers/ground-truths/find-ground-truths.ts
@@ -31,7 +31,11 @@ async function findChatBotTruths(
   params: AppParamsHelper<"chat-bot">,
   systemMsgObj: GroundTruthsSystemMessage<"chat-bot">
 ): Promise<string[]> {
-  const { adapters: { openai: { completions } } } = context;
+  const {
+    adapters: {
+      openai: { completions },
+    },
+  } = context;
   const systemMsg = createGroundTruthSysMsg(systemMsgObj);
   const truths = await completions.createGroundTruthCompletion<"chat-bot">(context, JSON.stringify(params), systemMsg, "o1-mini");
   return validateGroundTruths(truths);
@@ -42,7 +46,11 @@ async function findCodeReviewTruths(
   params: AppParamsHelper<"code-review">,
   systemMsgObj: GroundTruthsSystemMessage<"code-review">
 ): Promise<string[]> {
-  const { adapters: { openai: { completions } } } = context;
+  const {
+    adapters: {
+      openai: { completions },
+    },
+  } = context;
   const systemMsg = createGroundTruthSysMsg(systemMsgObj);
   const truths = await completions.createGroundTruthCompletion<"code-review">(context, params.taskSpecification, systemMsg, "gpt-4o");
   return validateGroundTruths(truths);
diff --git a/tests/main.test.ts b/tests/main.test.ts
index 57c0e72..caec32c 100644
--- a/tests/main.test.ts
+++ b/tests/main.test.ts
@@ -403,6 +403,9 @@ function createContext(body = TEST_SLASH_COMMAND) {
               },
             };
           },
+          createGroundTruthCompletion: async (): Promise<string> => {
+            return '["This is a mock answer for the chat"]';
+          },
         },
       },
     },

From 9615b00f9699e0b67e6f32b4ef9ce4a743414673 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 22:25:40 +0100
Subject: [PATCH 19/20] chore: remove t.ts gitignore

---
 .gitignore | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3dcd5ed..e23b105 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,5 +15,4 @@ junit.xml
 cypress/screenshots
 script.ts
 .wrangler
-test-dashboard.md
-t.ts
\ No newline at end of file
+test-dashboard.md
\ No newline at end of file

From 29177f5ffc2fd6a94c8098b4a61a18a6b5190e14 Mon Sep 17 00:00:00 2001
From: Keyrxng <106303466+Keyrxng@users.noreply.github.com>
Date: Fri, 25 Oct 2024 22:50:54 +0100
Subject: [PATCH 20/20] chore: embed groundTruths in html comment

---
 src/adapters/openai/helpers/completions.ts |  9 +++--
 src/plugin.ts                              | 39 ++++++++++++++++++++--
 tests/main.test.ts                         | 11 +++---
 3 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/src/adapters/openai/helpers/completions.ts b/src/adapters/openai/helpers/completions.ts
index cd15059..1d282b5 100644
--- a/src/adapters/openai/helpers/completions.ts
+++ b/src/adapters/openai/helpers/completions.ts
@@ -6,6 +6,7 @@ const MAX_TOKENS = 7000;
 
 export interface CompletionsType {
   answer: string;
+  groundTruths: string[];
   tokenUsage: {
     input: number;
     output: number;
@@ -73,9 +74,13 @@ export class Completions extends SuperOpenAi {
     });
     const answer = res.choices[0].message;
     if (answer && answer.content && res.usage) {
-      return { answer: answer.content, tokenUsage: { input: res.usage.prompt_tokens, output: res.usage.completion_tokens, total: res.usage.total_tokens } };
+      return {
+        answer: answer.content,
+        groundTruths,
+        tokenUsage: { input: res.usage.prompt_tokens, output: res.usage.completion_tokens, total: res.usage.total_tokens },
+      };
     }
-    return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 } };
+    return { answer: "", tokenUsage: { input: 0, output: 0, total: 0 }, groundTruths };
   }
 
   async createGroundTruthCompletion<TApp extends ModelApplications>(
diff --git a/src/plugin.ts b/src/plugin.ts
index b1ff223..a685297 100644
--- a/src/plugin.ts
+++ b/src/plugin.ts
@@ -57,13 +57,19 @@ export async function runPlugin(context: Context) {
   let commentToPost;
   try {
     const response = await askQuestion(context, question);
-    const { answer, tokenUsage } = response;
+    const { answer, tokenUsage, groundTruths } = response;
     if (!answer) {
       throw logger.error(`No answer from OpenAI`);
     }
     logger.info(`Answer: ${answer}`, { tokenUsage });
-    const tokens = `\n\n<!--\n${JSON.stringify(tokenUsage, null, 2)}\n--!>`;
-    commentToPost = answer + tokens;
+
+    const metadata = {
+      groundTruths,
+      tokenUsage,
+    };
+
+    const metadataString = createStructuredMetadata("LLM Ground Truths and Token Usage", logger.info(`Answer: ${answer}`, { metadata }));
+    commentToPost = answer + metadataString;
   } catch (err) {
     let errorMessage;
     if (err instanceof LogReturn) {
@@ -81,3 +87,30 @@ export async function runPlugin(context: Context) {
 function sanitizeMetadata(obj: LogReturn["metadata"]): string {
   return JSON.stringify(obj, null, 2).replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/--/g, "&#45;&#45;");
 }
+
+function createStructuredMetadata(header: string | undefined, logReturn: LogReturn) {
+  let logMessage, metadata;
+  if (logReturn) {
+    logMessage = logReturn.logMessage;
+    metadata = logReturn.metadata;
+  }
+
+  const jsonPretty = sanitizeMetadata(metadata);
+  const stackLine = new Error().stack?.split("\n")[2] ?? "";
+  const caller = stackLine.match(/at (\S+)/)?.[1] ?? "";
+  const ubiquityMetadataHeader = `\n\n<!-- Ubiquity - ${header} - ${caller} - ${metadata?.revision}`;
+
+  let metadataSerialized: string;
+  const metadataSerializedVisible = ["```json", jsonPretty, "```"].join("\n");
+  const metadataSerializedHidden = [ubiquityMetadataHeader, jsonPretty, "-->"].join("\n");
+
+  if (logMessage?.type === "fatal") {
+    // if the log message is fatal, then we want to show the metadata
+    metadataSerialized = [metadataSerializedVisible, metadataSerializedHidden].join("\n");
+  } else {
+    // otherwise we want to hide it
+    metadataSerialized = metadataSerializedHidden;
+  }
+
+  return metadataSerialized;
+}
diff --git a/tests/main.test.ts b/tests/main.test.ts
index caec32c..30e76b6 100644
--- a/tests/main.test.ts
+++ b/tests/main.test.ts
@@ -18,6 +18,7 @@ const TEST_SLASH_COMMAND = "@UbiquityOS what is pi?";
 const LOG_CALLER = "_Logs.<anonymous>";
 const ISSUE_ID_2_CONTENT = "More context here #2";
 const ISSUE_ID_3_CONTENT = "More context here #3";
+const MOCK_ANSWER = "This is a mock answer for the chat";
 
 type Comment = {
   id: number;
@@ -61,7 +62,7 @@ describe("Ask plugin tests", () => {
 
     expect(res).toBeDefined();
 
-    expect(res?.answer).toBe("This is a mock answer for the chat");
+    expect(res?.answer).toBe(MOCK_ANSWER);
   });
 
   it("should not ask GPT a question if comment is from a bot", async () => {
@@ -106,7 +107,6 @@ describe("Ask plugin tests", () => {
     createComments([transformCommentTemplate(1, 1, TEST_QUESTION, "ubiquity", "test-repo", true)]);
     await runPlugin(ctx);
 
-    expect(infoSpy).toHaveBeenCalledTimes(3);
     expect(infoSpy).toHaveBeenNthCalledWith(1, `Asking question: @UbiquityOS ${TEST_QUESTION}`);
     expect(infoSpy).toHaveBeenNthCalledWith(3, "Answer: This is a mock answer for the chat", {
       caller: LOG_CALLER,
@@ -130,8 +130,6 @@ describe("Ask plugin tests", () => {
 
     await runPlugin(ctx);
 
-    expect(infoSpy).toHaveBeenCalledTimes(3);
-
     expect(infoSpy).toHaveBeenNthCalledWith(1, `Asking question: @UbiquityOS ${TEST_QUESTION}`);
 
     const prompt = `=== Current Issue #1 Specification === ubiquity/test-repo/1 ===
@@ -395,7 +393,8 @@ function createContext(body = TEST_SLASH_COMMAND) {
         completions: {
           createCompletion: async (): Promise<CompletionsType> => {
             return {
-              answer: "This is a mock answer for the chat",
+              answer: MOCK_ANSWER,
+              groundTruths: [MOCK_ANSWER],
               tokenUsage: {
                 input: 1000,
                 output: 150,
@@ -404,7 +403,7 @@ function createContext(body = TEST_SLASH_COMMAND) {
             };
           },
           createGroundTruthCompletion: async (): Promise<string> => {
-            return '["This is a mock answer for the chat"]';
+            return `["${MOCK_ANSWER}"]`;
           },
         },
       },