allow script to build output as well (#998)

* allow script to build output as well * fix: 🐛 correct JSON.stringify usage in curl command * docs: ✏️ add note on upgrading OpenAI account * feat: ✨ add clearModelAlias to runtime host and update usage * feat: ✨ enhance output handling and add model alias reset * cleanup list and pull * added sript * refactor: 🔄 update LLM aliases in llms.json * refactor: ♻️ streamline model parsing and cleanup llms.json * fix pull * fix: 🐛 update alias for 'large' to use latest tag * style: ✏️ refine system prompt for conciseness * refactor: ✏️ clarify system prompt behavior * fix: 🐛 update model alias clearing logic * fix: 🐛 correct string template usage in fetch command * fix: 🐛 handle error objects with "error" property * smaller ollama model * feat: ✨ add JSONL parsing and error handling in pullModel function * merges * fix: 🐛 update clearModelAlias to prevent deletion error
microsoft · Jan 13, 2025 · fc583f9 · fc583f9
1 parent db4febd
commit fc583f9
Show file tree

Hide file tree

Showing 15 changed files with 206 additions and 30 deletions.
diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx
@@ -199,6 +199,13 @@ It uses the `OPENAI_API_...` environment variables.
 
 <ol>
 
+<li>
+
+[Upgrade your account](https://platform.openai.com/settings/organization/billing/overview) to get access to the models.
+You will get 404s if you do not have a paying account.
+
+</li>
+
 <li>
     Create a new secret key from the [OpenAI API Keys
     portal](https://platform.openai.com/api-keys).
@@ -261,7 +268,7 @@ This provider is useful for prototyping and subject to [rate limits](https://doc
 depending on your subscription.
 
 ```js "github:"
-script({ model: "github:gpt-4" })
+script({ model: "github:gpt-4o" })
 ```
 
 **If you are running from a [GitHub Codespace](https://github.com/features/codespaces), the token is already configured for you.**

diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts
@@ -121,6 +121,10 @@ export class NodeHost implements RuntimeHost {
         return Object.freeze(res)
     }
 
+    clearModelAlias(source: "cli" | "env" | "config" | "script") {
+        this._modelAliases[source] = {}
+    }
+
     setModelAlias(
         source: "cli" | "env" | "config" | "script",
         id: string,

diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts
@@ -148,6 +148,7 @@ export async function runScriptInternal(
 ): Promise<{ exitCode: number; result?: GenerationResult }> {
     const { trace = new MarkdownTrace(), infoCb, partialCb } = options || {}
 
+    runtimeHost.clearModelAlias("script")
     let result: GenerationResult
     const workspaceFiles = options.workspaceFiles
     const excludedFiles = options.excludedFiles

diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts
@@ -151,7 +151,7 @@ export async function callExpander(
 function traceEnv(
     model: string,
     trace: MarkdownTrace,
-    env: ExpansionVariables
+    env: Partial<ExpansionVariables>
 ) {
     trace.startDetails("🏡 env")
     trace.files(env.files, {

diff --git a/packages/core/src/host.ts b/packages/core/src/host.ts
@@ -145,6 +145,7 @@ export interface RuntimeHost extends Host {
         options?: TraceOptions & CancellationOptions
     ): Promise<ResponseStatus>
 
+    clearModelAlias(source: "cli" | "env" | "config" | "script"): void
     setModelAlias(
         source: "env" | "cli" | "config" | "script",
         id: string,

diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts
@@ -3,7 +3,7 @@
 // The context is essential for executing prompts within a project environment.
 
 import { host } from "./host"
-import { arrayify, dotGenaiscriptPath } from "./util"
+import { arrayify, assert, dotGenaiscriptPath } from "./util"
 import { runtimeHost } from "./host"
 import { MarkdownTrace } from "./trace"
 import { createParsers } from "./parsers"
@@ -45,9 +45,10 @@ export async function createPromptContext(
     options: GenerationOptions,
     model: string
 ) {
-    const { generator, vars, ...varsNoGenerator } = ev
+    const { generator, vars, output, ...varsNoGenerator } = ev
     // Clone variables to prevent modification of the original object
-    const env = { generator, vars, ...structuredClone(varsNoGenerator) }
+    const env = { generator, vars, output, ...structuredClone(varsNoGenerator) }
+    assert(!!output, "missing output")
     // Create parsers for the given trace and model
     const parsers = await createParsers({ trace, model })
     const path = runtimeHost.path
@@ -304,7 +305,7 @@ export async function createPromptContext(
     }
     env.generator = ctx
     env.vars = proxifyVars(env.vars)
-    ctx.env = Object.freeze(env)
+    ctx.env = Object.freeze<ExpansionVariables>(env as ExpansionVariables)
 
     return ctx
 }
diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts
@@ -1,9 +1,9 @@
 // Import necessary modules and functions for handling chat sessions, templates, file management, etc.
 import { executeChatSession, tracePromptResult } from "./chat"
 import { GenerationStatus, Project } from "./server/messages"
-import { arrayify, assert, relativePath } from "./util"
+import { arrayify, assert, logInfo, logVerbose, relativePath } from "./util"
 import { runtimeHost } from "./host"
-import { MarkdownTrace } from "./trace"
+import { MarkdownTrace, TraceChunkEvent } from "./trace"
 import { CORE_VERSION } from "./version"
 import { expandFiles } from "./fs"
 import { CSVToMarkdown } from "./csv"
@@ -17,6 +17,7 @@ import { parsePromptParameters } from "./vars"
 import { resolveFileContent } from "./file"
 import { expandTemplate } from "./expander"
 import { resolveLanguageModel } from "./lm"
+import { TRACE_CHUNK } from "./constants"
 import { checkCancelled } from "./cancellation"
 
 // Asynchronously resolve expansion variables needed for a template
@@ -34,8 +35,9 @@ async function resolveExpansionVars(
     trace: MarkdownTrace,
     template: PromptScript,
     fragment: Fragment,
+    output: OutputTrace,
     vars: Record<string, string | number | boolean | object>
-) {
+): Promise<ExpansionVariables> {
     const root = runtimeHost.projectFolder()
 
     const files: WorkspaceFile[] = []
@@ -95,7 +97,9 @@ async function resolveExpansionVars(
         meta,
         vars: attrs,
         secrets,
-    } satisfies Partial<ExpansionVariables>
+        output,
+        generator: undefined,
+    } as ExpansionVariables
     return res
 }
 
@@ -123,6 +127,10 @@ export async function runTemplate(
 
     runtimeHost.project = prj
 
+    const output = new MarkdownTrace()
+    output.addEventListener(TRACE_CHUNK, (e) =>
+        logVerbose((e as TraceChunkEvent).chunk)
+    )
     try {
         if (cliInfo) {
             trace.heading(3, `🧠 ${template.id}`)
@@ -135,6 +143,7 @@ export async function runTemplate(
             trace,
             template,
             fragment,
+            output,
             options.vars
         )
         let {
@@ -157,13 +166,7 @@ export async function runTemplate(
             logprobs,
             topLogprobs,
             disposables,
-        } = await expandTemplate(
-            prj,
-            template,
-            options,
-            vars as ExpansionVariables,
-            trace
-        )
+        } = await expandTemplate(prj, template, options, vars, trace)
 
         // Handle failed expansion scenario
         if (status !== "success" || !messages.length) {
@@ -175,7 +178,7 @@ export async function runTemplate(
                 vars,
                 label,
                 version,
-                text: "",
+                text: output.content,
                 edits: [],
                 annotations: [],
                 changelogs: [],
@@ -214,7 +217,7 @@ export async function runTemplate(
                 vars,
                 label,
                 version,
-                text: "",
+                text: output.content,
                 edits: [],
                 annotations: [],
                 changelogs: [],
@@ -245,7 +248,7 @@ export async function runTemplate(
             topLogprobs,
             stats: options.stats.createChild(connection.info.model),
         }
-        const output = await executeChatSession(
+        const chatResult = await executeChatSession(
             connection.configuration,
             cancellationToken,
             messages,
@@ -260,7 +263,7 @@ export async function runTemplate(
             disposables,
             genOptions
         )
-        tracePromptResult(trace, output)
+        tracePromptResult(trace, chatResult)
 
         const {
             json,
@@ -273,8 +276,12 @@ export async function runTemplate(
             fileEdits,
             changelogs,
             edits,
-        } = output
-        let { text, annotations } = output
+        } = chatResult
+        let { text, annotations } = chatResult
+        const userContent = output.content
+        if (userContent) {
+            text = userContent + "\n\n" + text
+        }
 
         // Reporting and tracing output
         if (fences?.length)
@@ -333,8 +340,8 @@ export async function runTemplate(
             genVars,
             schemas,
             json,
-            logprobs: output.logprobs,
-            perplexity: output.perplexity,
+            logprobs: chatResult.logprobs,
+            perplexity: chatResult.perplexity,
             stats: {
                 cost: options.stats.cost(),
                 ...options.stats.accumulatedUsage(),

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -32,6 +32,7 @@ import { consoleLogFormat, stdout } from "./logging"
 import { isGlobMatch } from "./glob"
 import {
     arrayify,
+    assert,
     deleteEmptyValues,
     logError,
     logVerbose,
@@ -87,7 +88,6 @@ import { videoExtractAudio } from "./ffmpeg"
 import { BufferToBlob } from "./bufferlike"
 import { host } from "./host"
 import { srtVttRender } from "./transcription"
-import { filenameOrFileToFilename } from "./unwrappers"
 
 export function createChatTurnGenerationContext(
     options: GenerationOptions,
@@ -321,6 +321,7 @@ export function createChatGenerationContext(
 ): RunPromptContextNode {
     const { cancellationToken, infoCb } = options || {}
     const { prj, env } = projectOptions
+    assert(!!env.output, "output missing")
     const turnCtx = createChatTurnGenerationContext(options, trace)
     const node = turnCtx.node
 

diff --git a/packages/core/src/testhost.ts b/packages/core/src/testhost.ts
@@ -79,6 +79,9 @@ export class TestHost implements RuntimeHost {
         return { ok: true }
     }
 
+    clearModelAlias(source: "cli" | "env" | "config" | "script"): void {
+        ;(this.modelAliases as any)[source] = {}
+    }
     setModelAlias(
         source: "cli" | "env" | "config",
         id: string,

diff --git a/packages/core/src/trace.ts b/packages/core/src/trace.ts
@@ -28,7 +28,7 @@ export class TraceChunkEvent extends Event {
     }
 }
 
-export class MarkdownTrace extends EventTarget implements ToolCallTrace {
+export class MarkdownTrace extends EventTarget implements OutputTrace {
     readonly _errors: { message: string; error: SerializedError }[] = []
     private detailsDepth = 0
     private _content: (string | MarkdownTrace)[] = []