feat(ts): prompts ts sdk

js/examples/apps/phoenix-experiment-runner/index.ts

@@ -1,10 +1,10 @@
 /* eslint-disable no-console */
+import { createClient } from "@arizeai/phoenix-client";
 import {
   asEvaluator,
-  createClient,
   runExperiment,
   type RunExperimentParams,
-} from "@arizeai/phoenix-client";
+} from "@arizeai/phoenix-client/experimental";
 import { intro, outro, select, spinner, log, confirm } from "@clack/prompts";
 import { Factuality, Humor } from "autoevals";
 import dotenv from "dotenv";

js/packages/phoenix-client/examples/apply_prompt.ts

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import OpenAI from "openai";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'\nAssumes that the prompt is openai with an openai model"
+  );
+}
+
+if (!OPENAI_API_KEY) {
+  throw new Error("OPENAI_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const openai = new OpenAI({
+  apiKey: OPENAI_API_KEY,
+});
+
+const main = async () => {
+  const prompt = await getPrompt({
+    client,
+    prompt: { name: PROMPT_NAME },
+  });
+
+  const openAIParams = toSDK({
+    prompt,
+    sdk: "openai",
+    variables: {
+      question,
+    },
+  });
+
+  if (!openAIParams) {
+    throw new Error("Prompt could not be converted to OpenAI params");
+  }
+
+  const response = await openai.chat.completions.create({
+    ...openAIParams,
+    stream: false,
+  });
+
+  console.log(response.choices[0]?.message.content);
+};
+
+main();

js/packages/phoenix-client/examples/apply_prompt_anthropic.ts

@@ -0,0 +1,124 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import { Anthropic } from "@anthropic-ai/sdk";
+import { PromptLike } from "../src/types/prompts";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const PROMPT_TAG = process.env.PROMPT_TAG!;
+const PROMPT_VERSION_ID = process.env.PROMPT_VERSION_ID!;
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt_anthropic.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'"
+  );
+}
+
+if (!ANTHROPIC_API_KEY) {
+  throw new Error("ANTHROPIC_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const anthropic = new Anthropic({
+  apiKey: ANTHROPIC_API_KEY,
+});
+
+const main = async () => {
+  const promptArgument: PromptLike | null = PROMPT_VERSION_ID
+    ? { versionId: PROMPT_VERSION_ID }
+    : PROMPT_TAG && PROMPT_NAME
+      ? { name: PROMPT_NAME, tag: PROMPT_TAG }
+      : PROMPT_NAME
+        ? { name: PROMPT_NAME }
+        : null;
+  if (!promptArgument) {
+    throw new Error(
+      `Either PROMPT_VERSION_ID, PROMPT_TAG and PROMPT_NAME, or PROMPT_NAME must be provided in the environment`
+    );
+  }
+  console.log(`Getting prompt ${PROMPT_VERSION_ID}`);
+
+  // TODO: Apply variable replacement to the prompt
+  const prompt = await getPrompt({
+    client,
+    prompt: promptArgument,
+  });
+
+  if (!prompt) {
+    throw new Error("Prompt not found");
+  }
+
+  console.log(
+    `Loaded prompt: ${prompt.id}\n${prompt.description ? `\n${prompt.description}` : ""}`
+  );
+
+  console.log(`Converting prompt to OpenAI params`);
+
+  const anthropicParams = toSDK({
+    prompt,
+    sdk: "anthropic",
+    variables: {
+      question,
+    },
+  });
+
+  if (!anthropicParams) {
+    throw new Error("Prompt could not be converted to Anthropic params");
+  }
+
+  // @ts-expect-error Anthropic doesn't support these parameters
+  delete anthropicParams.frequency_penalty;
+  // @ts-expect-error Anthropic doesn't support these parameters
+  delete anthropicParams.presence_penalty;
+
+  console.log(`Applying prompt to Anthropic`);
+  const response = await anthropic.messages.create({
+    ...anthropicParams,
+    // we may not have an anthropic model saved in the prompt
+    model: "claude-3-5-sonnet-20240620",
+    // TODO: should this be strongly typed inside of toSDK results if sdk: "anthropic"?
+    stream: true,
+  });
+
+  console.log(`Streaming response from OpenAI:\n\n`);
+
+  let responseText = "";
+  let responseJson = "";
+  for await (const chunk of response) {
+    if (chunk.type === "message_delta") {
+      console.clear();
+      console.log("Input:\n");
+      console.log(JSON.stringify(anthropicParams.messages, null, 2));
+      console.log("\nOutput:\n");
+      try {
+        console.log(JSON.stringify(JSON.parse(responseText), null, 2));
+        console.log(JSON.stringify(JSON.parse(responseJson), null, 2));
+      } catch {
+        console.log(responseText);
+        console.log(responseJson);
+      }
+    } else if (chunk.type === "content_block_delta") {
+      console.clear();
+      if (chunk.delta.type === "text_delta") {
+        responseText += String(chunk.delta.text);
+      }
+      if (chunk.delta.type === "input_json_delta") {
+        responseJson += chunk.delta.partial_json;
+      }
+      console.log(responseText);
+    }
+  }
+
+  console.log("\n\n");
+  console.log(`Done!`);
+};
+
+main();

js/packages/phoenix-client/examples/apply_prompt_openai.ts

@@ -0,0 +1,110 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import OpenAI from "openai";
+import { PromptLike } from "../src/types/prompts";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const PROMPT_TAG = process.env.PROMPT_TAG!;
+const PROMPT_VERSION_ID = process.env.PROMPT_VERSION_ID!;
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt_openai.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'"
+  );
+}
+
+if (!OPENAI_API_KEY) {
+  throw new Error("OPENAI_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const openai = new OpenAI({
+  apiKey: OPENAI_API_KEY,
+});
+
+const main = async () => {
+  const promptArgument: PromptLike | null = PROMPT_VERSION_ID
+    ? { versionId: PROMPT_VERSION_ID }
+    : PROMPT_TAG && PROMPT_NAME
+      ? { name: PROMPT_NAME, tag: PROMPT_TAG }
+      : PROMPT_NAME
+        ? { name: PROMPT_NAME }
+        : null;
+  if (!promptArgument) {
+    throw new Error(
+      `Either PROMPT_VERSION_ID, PROMPT_TAG and PROMPT_NAME, or PROMPT_NAME must be provided in the environment`
+    );
+  }
+  console.log(`Getting prompt ${PROMPT_VERSION_ID}`);
+
+  // TODO: Apply variable replacement to the prompt
+  const prompt = await getPrompt({
+    client,
+    prompt: promptArgument,
+  });
+
+  if (!prompt) {
+    throw new Error("Prompt not found");
+  }
+
+  console.log(
+    `Loaded prompt: ${prompt.id}\n${prompt.description ? `\n${prompt.description}` : ""}`
+  );
+
+  console.log(`Converting prompt to OpenAI params`);
+
+  const openAIParams = toSDK({
+    prompt,
+    sdk: "openai",
+    variables: {
+      question,
+    },
+  });
+
+  if (!openAIParams) {
+    throw new Error("Prompt could not be converted to OpenAI params");
+  }
+
+  console.log(`Applying prompt to OpenAI`);
+  const response = await openai.chat.completions.create({
+    ...openAIParams,
+    // we may not have an openai model saved in the prompt
+    model: "gpt-4o-mini",
+    // TODO: should this be strongly typed inside of toSDK results if sdk: "openai"?
+    stream: true,
+  });
+
+  console.log(`Streaming response from OpenAI:\n\n`);
+
+  let responseText = "";
+  for await (const chunk of response) {
+    if (chunk.choices[0]?.delta?.content) {
+      responseText += chunk.choices[0]?.delta?.content;
+      console.clear();
+      console.log("Input:\n");
+      console.log(JSON.stringify(openAIParams.messages, null, 2));
+      console.log("\nOutput:\n");
+      try {
+        console.log(JSON.stringify(JSON.parse(responseText), null, 2));
+      } catch {
+        console.log(responseText);
+      }
+    } else if (chunk.choices[0]?.delta?.tool_calls) {
+      console.log(chunk.choices[0]?.delta?.tool_calls);
+    }
+  }
+
+  console.log("\n\n");
+  console.log(`Done!`);
+};
+
+main();

js/packages/phoenix-client/package.json

@@ -9,6 +9,22 @@
     ".": {
       "import": "./dist/esm/index.js",
       "require": "./dist/src/index.js"
+    },
+    "./prompts": {
+      "import": "./dist/esm/prompts/index.js",
+      "require": "./dist/src/prompts/index.js"
+    },
+    "./experimental": {
+      "import": "./dist/esm/experimental/index.js",
+      "require": "./dist/src/experimental/index.js"
+    },
+    "./utils/*": {
+      "import": "./dist/esm/utils/*.js",
+      "require": "./dist/src/utils/*.js"
+    },
+    "./types/*": {
+      "import": "./dist/esm/types/*.d.ts",
+      "require": "./dist/src/types/*.d.ts"
     }
   },
   "files": [
@@ -23,23 +39,30 @@
     "build": "tsc --build tsconfig.json tsconfig.esm.json && tsc-alias -p tsconfig.esm.json",
     "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json && rimraf dist/test dist/examples",
     "type:check": "tsc --noEmit",
-    "test": "vitest run"
+    "test": "vitest"
   },
   "keywords": [],
   "author": "",
   "license": "ELv2",
   "devDependencies": {
+    "@anthropic-ai/sdk": "^0.35.0",
     "@types/node": "^20.14.11",
     "openapi-typescript": "^7.4.1",
-    "tsx": "^4.19.1"
+    "tsx": "^4.19.1",
+    "vitest": "^2.1.8"
   },
   "dependencies": {
     "openapi-fetch": "^0.12.2",
     "tiny-invariant": "^1.3.3",
-    "zod": "^3.23.8"
+    "zod": "^3.24.1",
+    "zod-to-json-schema": "^3.24.1"
   },
   "packageManager": "[email protected]",
   "engines": {
     "node": ">=18"
+  },
+  "peerDependencies": {
+    "ai": "^4.1.0",
+    "openai": "^4.77.0"
   }
 }