microsoft · pelikhan · Jan 24, 2025
diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts
@@ -1,6 +1,11 @@
 // cspell: disable
 import { MarkdownTrace, TraceOptions } from "./trace"
-import { PromptImage, PromptPrediction, renderPromptNode } from "./promptdom"
+import {
+    PromptAudio,
+    PromptImage,
+    PromptPrediction,
+    renderPromptNode,
+} from "./promptdom"
 import { host, runtimeHost } from "./host"
 import { GenerationOptions } from "./generation"
 import { dispose } from "./dispose"
@@ -46,6 +51,7 @@ import { parseModelIdentifier, traceLanguageModelConnection } from "./models"
 import {
     ChatCompletionAssistantMessageParam,
     ChatCompletionContentPartImage,
+    ChatCompletionContentPartInputAudio,
     ChatCompletionMessageParam,
     ChatCompletionResponse,
     ChatCompletionsOptions,
@@ -95,9 +101,11 @@ import { deleteUndefinedValues } from "./cleaners"
 
 export function toChatCompletionUserMessage(
     expanded: string,
-    images?: PromptImage[]
+    images?: PromptImage[],
+    audios?: PromptAudio[]
 ): ChatCompletionUserMessageParam {
     const imgs = images?.filter(({ url }) => url) || []
+    const auds = audios?.filter(({ data }) => data) || []
     if (imgs.length)
         return <ChatCompletionUserMessageParam>{
             role: "user",
@@ -108,13 +116,23 @@ export function toChatCompletionUserMessage(
                 },
                 ...imgs.map(
                     ({ url, detail }) =>
-                        <ChatCompletionContentPartImage>{
+                        ({
                             type: "image_url",
                             image_url: {
                                 url,
                                 detail,
                             },
-                        }
+                        }) satisfies ChatCompletionContentPartImage
+                ),
+                ...auds.map(
+                    ({ data, format }) =>
+                        ({
+                            type: "input_audio",
+                            input_audio: {
+                                data,
+                                format,
+                            },
+                        }) satisfies ChatCompletionContentPartInputAudio
                 ),
             ],
         }
@@ -135,9 +153,11 @@ export type ChatCompletionHandler = (
 export type ListModelsFunction = (
     cfg: LanguageModelConfiguration,
     options: TraceOptions & CancellationOptions
-) => Promise<ResponseStatus & {
-    models?: LanguageModelInfo[]
-}>
+) => Promise<
+    ResponseStatus & {
+        models?: LanguageModelInfo[]
+    }
+>
 
 export type PullModelFunction = (
     cfg: LanguageModelConfiguration,

diff --git a/packages/core/src/chattypes.ts b/packages/core/src/chattypes.ts
@@ -100,6 +100,9 @@ export type ChatCompletionUserMessageParam =
 export type ChatCompletionContentPartImage =
     OpenAI.Chat.Completions.ChatCompletionContentPartImage
 
+export type ChatCompletionContentPartInputAudio =
+    OpenAI.Chat.Completions.ChatCompletionContentPartInputAudio
+
 // Parameters for creating embeddings
 export type EmbeddingCreateParams = OpenAI.Embeddings.EmbeddingCreateParams
 

diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts
@@ -10,6 +10,7 @@ import {
 } from "./constants"
 import {
     finalizeMessages,
+    PromptAudio,
     PromptImage,
     PromptPrediction,
     renderPromptNode,
@@ -50,6 +51,7 @@ export async function callExpander(
     let logs = ""
     let messages: ChatCompletionMessageParam[] = []
     let images: PromptImage[] = []
+    let audios: PromptAudio[] = []
     let schemas: Record<string, JSONSchema> = {}
     let functions: ToolCallback[] = []
     let fileMerges: FileMergeHandler[] = []
@@ -82,6 +84,7 @@ export async function callExpander(
             const {
                 messages: msgs,
                 images: imgs,
+                audios: auds,
                 errors,
                 schemas: schs,
                 functions: fns,
@@ -98,6 +101,7 @@ export async function callExpander(
             })
             messages = msgs
             images = imgs
+            audios = auds
             schemas = schs
             functions = fns
             fileMerges = fms
@@ -136,6 +140,7 @@ export async function callExpander(
         statusText,
         messages,
         images,
+        audios,
         schemas,
         functions: Object.freeze(functions),
         fileMerges,
@@ -247,6 +252,7 @@ export async function expandTemplate(
 
     const { status, statusText, messages } = prompt
     const images = prompt.images.slice(0)
+    const audios = prompt.audios.slice(0)
     const schemas = structuredClone(prompt.schemas)
     const tools = prompt.functions.slice(0)
     const fileMerges = prompt.fileMerges.slice(0)
@@ -279,8 +285,8 @@ export async function expandTemplate(
         }
     }
 
-    if (prompt.images?.length)
-        messages.push(toChatCompletionUserMessage("", prompt.images))
+    if (images?.length || audios?.length)
+        messages.push(toChatCompletionUserMessage("", images, audios))
     if (prompt.aici) messages.push(prompt.aici)
 
     const addSystemMessage = (content: string) => {
@@ -314,6 +320,7 @@ export async function expandTemplate(
             const sysr = await callExpander(prj, system, env, trace, options)
 
             if (sysr.images) images.push(...sysr.images)
+            if (sysr.audios) audios.push(...sysr.audios)
             if (sysr.schemas) Object.assign(schemas, sysr.schemas)
             if (sysr.functions) tools.push(...sysr.functions)
             if (sysr.fileMerges) fileMerges.push(...sysr.fileMerges)
@@ -394,6 +401,7 @@ ${schemaTs}
         cache,
         messages,
         images,
+        audios,
         schemas,
         tools,
         status: <GenerationStatus>status,

diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts
@@ -45,6 +45,7 @@ export interface PromptNode extends ContextExpansionOptions {
     type?:
         | "text"
         | "image"
+        | "audio"
         | "schema"
         | "tool"
         | "fileMerge"
@@ -150,6 +151,18 @@ export interface PromptImageNode extends PromptNode {
     resolved?: PromptImage // Resolved image information
 }
 
+export interface PromptAudio {
+    filename?: string
+    data: string
+    format: "mp3" | "wav"
+}
+
+export interface PromptAudioNode extends PromptNode {
+    type: "audio"
+    value: Awaitable<PromptAudio> // Image information
+    resolved?: PromptAudio // Resolved image information
+}
+
 // Interface for a schema node.
 export interface PromptSchemaNode extends PromptNode {
     type: "schema"
@@ -418,6 +431,15 @@ export function createImageNode(
     return { type: "image", value, ...(options || {}) }
 }
 
+// Function to create an image node.
+export function createAudioNode(
+    value: Awaitable<PromptAudio>,
+    options?: ContextExpansionOptions
+): PromptAudioNode {
+    assert(value !== undefined)
+    return { type: "audio", value, ...(options || {}) }
+}
+
 // Function to create a schema node.
 export function createSchemaNode(
     name: string,
@@ -556,6 +578,7 @@ export interface PromptNodeVisitor {
     def?: (node: PromptDefNode) => Awaitable<void> // Definition node visitor
     defData?: (node: PromptDefDataNode) => Awaitable<void> // Definition data node visitor
     image?: (node: PromptImageNode) => Awaitable<void> // Image node visitor
+    audio?: (node: PromptAudioNode) => Awaitable<void> // Audio node visitor
     schema?: (node: PromptSchemaNode) => Awaitable<void> // Schema node visitor
     tool?: (node: PromptToolNode) => Awaitable<void> // Function node visitor
     fileMerge?: (node: PromptFileMergeNode) => Awaitable<void> // File merge node visitor
@@ -585,6 +608,9 @@ export async function visitNode(node: PromptNode, visitor: PromptNodeVisitor) {
         case "image":
             await visitor.image?.(node as PromptImageNode)
             break
+        case "audio":
+            await visitor.audio?.(node as PromptAudioNode)
+            break
         case "schema":
             await visitor.schema?.(node as PromptSchemaNode)
             break
@@ -632,6 +658,7 @@ export async function visitNode(node: PromptNode, visitor: PromptNodeVisitor) {
 // Interface for representing a rendered prompt node.
 export interface PromptNodeRender {
     images: PromptImage[] // Images included in the prompt
+    audios: PromptAudio[]
     errors: unknown[] // Errors encountered during rendering
     schemas: Record<string, JSONSchema> // Schemas included in the prompt
     functions: ToolCallback[] // Functions included in the prompt
@@ -847,6 +874,15 @@ async function resolvePromptNode(
                 n.error = e
             }
         },
+        audio: async (n) => {
+            try {
+                const v = await n.value
+                n.resolved = v
+                n.preview = n.resolved ? `<audio />` : undefined
+            } catch (e) {
+                n.error = e
+            }
+        },
     })
     return { errors: err }
 }
@@ -1186,6 +1222,7 @@ export async function renderPromptNode(
     ) => appendAssistantMessage(messages, content, options)
 
     const images: PromptImage[] = []
+    const audios: PromptAudio[] = []
     const errors: unknown[] = []
     const schemas: Record<string, JSONSchema> = {}
     const tools: ToolCallback[] = []
@@ -1248,6 +1285,16 @@ export async function renderPromptNode(
                 }
             }
         },
+        audio: async (n) => {
+            const value = n.resolved
+            if (value?.data) {
+                audios.push(value)
+                if (trace) {
+                    trace.startDetails(`🎤 audio ${value.filename || ""}`)
+                    trace.endDetails()
+                }
+            }
+        },
         schema: (n) => {
             const { name: schemaName, value: schema, options } = n
             if (schemas[schemaName])
@@ -1333,6 +1380,7 @@ ${trimNewlines(schemaText)}
 
     const res = Object.freeze<PromptNodeRender>({
         images,
+        audios,
         schemas,
         functions: tools,
         fileMerges,

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -938,6 +938,8 @@ export function createChatGenerationContext(
                         )
                         if (sysr.images?.length)
                             throw new NotSupportedError("images")
+                        if (sysr.audios?.length)
+                            throw new NotSupportedError("audios")
                         if (sysr.schemas) Object.assign(schemas, sysr.schemas)
                         if (sysr.functions) tools.push(...sysr.functions)
                         if (sysr.fileMerges?.length)

diff --git a/packages/vscode/src/lmaccess.ts b/packages/vscode/src/lmaccess.ts
@@ -55,6 +55,11 @@ function messagesToChatMessages(messages: ChatCompletionMessageParam[]) {
                     m.content.some((c) => c.type === "image_url")
                 )
                     throw new Error("Vision model not supported")
+                if (
+                    Array.isArray(m.content) &&
+                    m.content.some((c) => c.type === "input_audio")
+                )
+                    throw new Error("Ayudiuo model not supported")
                 return vscode.LanguageModelChatMessage.User(
                     renderMessageContent(m),
                     "genaiscript"