feat: Evals

src/evals/general/index.ts

@@ -0,0 +1,36 @@
+import { type Example, Run } from "langsmith";
+import { evaluate, EvaluationResult } from "langsmith/evaluation";
+import "dotenv/config";
+import { generatePostGraph } from "../../agent/subgraphs/generate-post/graph.js";
+
+const runGraph = async (
+  input: Record<string, any>
+): Promise<Record<string, any>> => {
+  return await generatePostGraph.invoke(input);
+};
+
+const evaluatePost = (run: Run, example?: Example): EvaluationResult => {
+  if (!example) {
+    throw new Error("No example provided");
+  }
+  if (!example.outputs) {
+    throw new Error("No example outputs provided");
+  }
+  if (!run.outputs) {
+    throw new Error("No run outputs provided");
+  }
+
+  // TODO: Implement evaluation logic
+  throw new Error("Evaluation logic not implemented");
+};
+
+async function runEval() {
+  const datasetName = "sma:generate-post:general";
+  await evaluate(runGraph, {
+    data: datasetName,
+    evaluators: [evaluatePost],
+    experimentPrefix: "Post Generation - General",
+  });
+}
+
+runEval();

src/evals/github/index.ts

@@ -0,0 +1,36 @@
+import { type Example, Run } from "langsmith";
+import { evaluate, EvaluationResult } from "langsmith/evaluation";
+import "dotenv/config";
+import { generatePostGraph } from "../../agent/subgraphs/generate-post/graph.js";
+
+const runGraph = async (
+  input: Record<string, any>
+): Promise<Record<string, any>> => {
+  return await generatePostGraph.invoke(input);
+};
+
+const evaluatePost = (run: Run, example?: Example): EvaluationResult => {
+  if (!example) {
+    throw new Error("No example provided");
+  }
+  if (!example.outputs) {
+    throw new Error("No example outputs provided");
+  }
+  if (!run.outputs) {
+    throw new Error("No run outputs provided");
+  }
+
+  // TODO: Implement evaluation logic
+  throw new Error("Evaluation logic not implemented");
+};
+
+async function runEval() {
+  const datasetName = "sma:generate-post:github";
+  await evaluate(runGraph, {
+    data: datasetName,
+    evaluators: [evaluatePost],
+    experimentPrefix: "Post Generation - Github",
+  });
+}
+
+runEval();

src/evals/twitter/index.ts

@@ -0,0 +1,36 @@
+import { type Example, Run } from "langsmith";
+import { evaluate, EvaluationResult } from "langsmith/evaluation";
+import "dotenv/config";
+import { generatePostGraph } from "../../agent/subgraphs/generate-post/graph.js";
+
+const runGraph = async (
+  input: Record<string, any>
+): Promise<Record<string, any>> => {
+  return await generatePostGraph.invoke(input);
+};
+
+const evaluatePost = (run: Run, example?: Example): EvaluationResult => {
+  if (!example) {
+    throw new Error("No example provided");
+  }
+  if (!example.outputs) {
+    throw new Error("No example outputs provided");
+  }
+  if (!run.outputs) {
+    throw new Error("No run outputs provided");
+  }
+
+  // TODO: Implement evaluation logic
+  throw new Error("Evaluation logic not implemented");
+};
+
+async function runEval() {
+  const datasetName = "sma:generate-post:twitter";
+  await evaluate(runGraph, {
+    data: datasetName,
+    evaluators: [evaluatePost],
+    experimentPrefix: "Post Generation - Twitter",
+  });
+}
+
+runEval();

src/evals/youtube/index.ts

@@ -0,0 +1,36 @@
+import { type Example, Run } from "langsmith";
+import { evaluate, EvaluationResult } from "langsmith/evaluation";
+import "dotenv/config";
+import { generatePostGraph } from "../../agent/subgraphs/generate-post/graph.js";
+
+const runGraph = async (
+  input: Record<string, any>
+): Promise<Record<string, any>> => {
+  return await generatePostGraph.invoke(input);
+};
+
+const evaluatePost = (run: Run, example?: Example): EvaluationResult => {
+  if (!example) {
+    throw new Error("No example provided");
+  }
+  if (!example.outputs) {
+    throw new Error("No example outputs provided");
+  }
+  if (!run.outputs) {
+    throw new Error("No run outputs provided");
+  }
+
+  // TODO: Implement evaluation logic
+  throw new Error("Evaluation logic not implemented");
+};
+
+async function runEval() {
+  const datasetName = "sma:generate-post:youtube";
+  await evaluate(runGraph, {
+    data: datasetName,
+    evaluators: [evaluatePost],
+    experimentPrefix: "Post Generation - YouTube",
+  });
+}
+
+runEval();