Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(playground): ensure round robin behavior on chat completion over datasets #5825

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

fix(playground): ensure round robin behavior on chat completion over datasets #5825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81b2b11
remove template options
axiomofjoy Dec 24, 2024
f0445bd
refactor input types
axiomofjoy Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions app/schema.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ input ChatCompletionInput {
invocationParameters: [InvocationParameterInput!]! = []
tools: [JSON!]
apiKey: String = null
template: TemplateOptions
templateLanguage: TemplateLanguage
templateVariables: JSON
}

input ChatCompletionMessageInput {
Expand Down Expand Up @@ -144,15 +145,13 @@ type ChatCompletionMutationPayload {

union ChatCompletionMutationPayloadChatCompletionMutationError = ChatCompletionMutationPayload | ChatCompletionMutationError

input ChatCompletionOverDatasetInput {
input ChatCompletionOverDatasetConfig {
messages: [ChatCompletionMessageInput!]!
model: GenerativeModelInput!
invocationParameters: [InvocationParameterInput!]! = []
tools: [JSON!]
apiKey: String = null
templateLanguage: TemplateLanguage!
datasetId: GlobalID!
datasetVersionId: GlobalID = null
experimentName: String = null
experimentDescription: String = null
experimentMetadata: JSON = {}
Expand All @@ -172,20 +171,24 @@ type ChatCompletionOverDatasetMutationPayload {
}

type ChatCompletionSubscriptionError implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
message: String!
}

type ChatCompletionSubscriptionExperiment implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
experiment: Experiment!
}

interface ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
}

type ChatCompletionSubscriptionResult implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
span: Span
experimentRun: ExperimentRun
Expand All @@ -196,6 +199,12 @@ type ChatCompletionToolCall {
function: ChatCompletionFunctionCall!
}

input ChatCompletionsOverDatasetInput {
configs: [ChatCompletionOverDatasetConfig!]!
datasetId: GlobalID!
datasetVersionId: GlobalID = null
}

input ClearProjectInput {
id: GlobalID!

Expand Down Expand Up @@ -938,6 +947,7 @@ type FloatInvocationParameter implements InvocationParameterBase {
}

type FunctionCallChunk implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
name: String!
arguments: String!
Expand Down Expand Up @@ -1170,7 +1180,7 @@ type Mutation {
patchUser(input: PatchUserInput!): UserMutationPayload!
patchViewer(input: PatchViewerInput!): UserMutationPayload!
deleteUsers(input: DeleteUsersInput!): Void
chatCompletionOverDataset(input: ChatCompletionOverDatasetInput!): ChatCompletionOverDatasetMutationPayload!
chatCompletionsOverDataset(input: ChatCompletionsOverDatasetInput!): ChatCompletionOverDatasetMutationPayload!
chatCompletion(input: ChatCompletionInput!): ChatCompletionMutationPayload!
}

Expand Down Expand Up @@ -1675,7 +1685,7 @@ type StringListInvocationParameter implements InvocationParameterBase {

type Subscription {
chatCompletion(input: ChatCompletionInput!): ChatCompletionSubscriptionPayload!
chatCompletionOverDataset(input: ChatCompletionOverDatasetInput!): ChatCompletionSubscriptionPayload!
chatCompletionsOverDataset(input: ChatCompletionsOverDatasetInput!): ChatCompletionSubscriptionPayload!
}

type SystemApiKey implements ApiKey & Node {
Expand All @@ -1701,12 +1711,8 @@ enum TemplateLanguage {
F_STRING
}

input TemplateOptions {
variables: JSON!
language: TemplateLanguage!
}

type TextChunk implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
content: String!
}
Expand Down Expand Up @@ -1735,6 +1741,7 @@ type TokenUsage {
}

type ToolCallChunk implements ChatCompletionSubscriptionPayload {
configIndex: Int
datasetExampleId: GlobalID
id: String!
function: FunctionCallChunk!
Expand Down
16 changes: 8 additions & 8 deletions app/src/pages/playground/PlaygroundDatasetExamplesTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ import {
const PAGE_SIZE = 10;

type ChatCompletionOverDatasetMutationPayload = Extract<
PlaygroundDatasetExamplesTableMutation$data["chatCompletionOverDataset"],
PlaygroundDatasetExamplesTableMutation$data["chatCompletionsOverDataset"],
{ __typename: "ChatCompletionOverDatasetMutationPayload" }
>;

Expand Down Expand Up @@ -474,7 +474,7 @@ export function PlaygroundDatasetExamplesTable({
if (response == null) {
return;
}
const chatCompletion = response.chatCompletionOverDataset;
const chatCompletion = response.chatCompletionsOverDataset;
switch (chatCompletion.__typename) {
case "ChatCompletionSubscriptionExperiment":
updateInstance({
Expand Down Expand Up @@ -565,13 +565,13 @@ export function PlaygroundDatasetExamplesTable({
updateInstance({
instanceId,
patch: {
experimentId: response.chatCompletionOverDataset.experimentId,
experimentId: response.chatCompletionsOverDataset.experimentId,
},
});
setExampleDataForInstance({
instanceId,
data: createExampleResponsesForInstance(
response.chatCompletionOverDataset
response.chatCompletionsOverDataset
),
});
},
Expand Down Expand Up @@ -966,9 +966,9 @@ export function PlaygroundDatasetExamplesTable({

graphql`
subscription PlaygroundDatasetExamplesTableSubscription(
$input: ChatCompletionOverDatasetInput!
$input: ChatCompletionsOverDatasetInput!
) {
chatCompletionOverDataset(input: $input) {
chatCompletionsOverDataset(input: $input) {
__typename
... on TextChunk {
content
Expand Down Expand Up @@ -1016,9 +1016,9 @@ graphql`

graphql`
mutation PlaygroundDatasetExamplesTableMutation(
$input: ChatCompletionOverDatasetInput!
$input: ChatCompletionsOverDatasetInput!
) {
chatCompletionOverDataset(input: $input) {
chatCompletionsOverDataset(input: $input) {
__typename
experimentId
examples {
Expand Down
4 changes: 2 additions & 2 deletions app/src/pages/playground/PlaygroundDatasetExamplesTableContext.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ import { PartialOutputToolCall } from "./PlaygroundToolCall";
type InstanceId = number;
export type ExampleId = string;
type ChatCompletionSubscriptionResult = Extract<
PlaygroundDatasetExamplesTableSubscription$data["chatCompletionOverDataset"],
PlaygroundDatasetExamplesTableSubscription$data["chatCompletionsOverDataset"],
{ __typename: "ChatCompletionSubscriptionResult" }
>;
export type Span = NonNullable<ChatCompletionSubscriptionResult["span"]>;

type ToolCallChunk = Extract<
PlaygroundDatasetExamplesTableSubscription$data["chatCompletionOverDataset"],
PlaygroundDatasetExamplesTableSubscription$data["chatCompletionsOverDataset"],
{ __typename: "ToolCallChunk" }
>;

Expand Down
21 changes: 12 additions & 9 deletions app/src/pages/playground/__generated__/PlaygroundDatasetExamplesTableMutation.graphql.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 12 additions & 9 deletions app/src/pages/playground/__generated__/PlaygroundDatasetExamplesTableSubscription.graphql.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 3 additions & 6 deletions app/src/pages/playground/__generated__/PlaygroundOutputMutation.graphql.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 3 additions & 6 deletions app/src/pages/playground/__generated__/PlaygroundOutputSubscription.graphql.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading