Skip to content

Commit

Permalink
Surface FCS badge.
Browse files Browse the repository at this point in the history
  • Loading branch information
cjcenizal committed Apr 18, 2024
1 parent 6fb8a3c commit 415e994
Show file tree
Hide file tree
Showing 14 changed files with 382 additions and 17 deletions.
10 changes: 9 additions & 1 deletion src/components/ChatItem.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Props = {
question?: string;
answer?: string;
searchResults?: DeserializedSearchResult[];
factualConsistencyScore?: React.ReactNode;
onRetry?: () => void;
isStreaming?: boolean;
};
Expand All @@ -47,7 +48,7 @@ type Props = {
* Defaults to showing just the user-supplied message, if it has not yet been answered.
* Otherwise, shows both question and answer, plus applicable references.
*/
export const ChatItem = ({ question, answer, searchResults, onRetry, isStreaming }: Props) => {
export const ChatItem = ({ question, answer, searchResults, factualConsistencyScore, onRetry, isStreaming }: Props) => {
const [isReferencesOpen, setIsReferencesOpen] = useState(false);
let content;

Expand Down Expand Up @@ -114,6 +115,13 @@ export const ChatItem = ({ question, answer, searchResults, onRetry, isStreaming
)}
</VuiText>

{factualConsistencyScore && (
<>
<VuiSpacer size="xs" />
{factualConsistencyScore}
</>
)}

{reorderedSearchResults && reorderedSearchResults.length > 0 && (
<>
<VuiSpacer size="s" />
Expand Down
26 changes: 21 additions & 5 deletions src/components/ChatView.tsx
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { Fragment, ReactNode, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { VuiButtonSecondary, VuiFlexContainer, VuiFlexItem, VuiSpacer } from "../vui";
import { Fragment, ReactNode, useEffect, useMemo, useRef, useState } from "react";
import { VuiButtonSecondary, VuiFlexContainer, VuiFlexItem, VuiSpacer, VuiContextProvider } from "../vui";
import { QueryInput } from "./QueryInput";
import { ChatItem } from "./ChatItem";
import { useChat } from "../useChat";
import { Loader } from "./Loader";
import { ChatBubbleIcon, MinimizeIcon } from "./Icons";
import { FactualConsistencyBadge } from "./FactualConsistencyBadge";
import { SummaryLanguage } from "types";

const inputSizeToQueryInputSize = {
Expand Down Expand Up @@ -142,15 +143,23 @@ export const ChatView = ({
const historyItems = useMemo(
() =>
messageHistory.map((turn, index) => {
const { question, answer, results } = turn;
const { question, answer, results, factualConsistencyScore } = turn;
const onRetry =
hasError && index === messageHistory.length - 1
? () => sendMessage({ query: question, isRetry: true })
: undefined;

return (
<Fragment key={index}>
<ChatItem question={question} answer={answer} searchResults={results} onRetry={onRetry} />
<ChatItem
question={question}
answer={answer}
searchResults={results}
factualConsistencyScore={
enableFactualConsistencyScore && <FactualConsistencyBadge score={factualConsistencyScore} />
}
onRetry={onRetry}
/>
{index < messageHistory.length - 1 && <VuiSpacer size="m" />}
</Fragment>
);
Expand All @@ -171,7 +180,7 @@ export const ChatView = ({

useEffect(updateScrollPosition, [isLoading, activeMessage]);

return isOpen ? (
const content = isOpen ? (
<div className="vrcbChatbotWrapper" style={{ zIndex }}>
<VuiFlexContainer className="vrcbHeader" spacing="none" direction="row">
<VuiFlexItem grow={1} alignItems="center">
Expand Down Expand Up @@ -201,6 +210,11 @@ export const ChatView = ({
question={activeMessage.question}
answer={activeMessage.answer}
searchResults={activeMessage.results}
factualConsistencyScore={
enableFactualConsistencyScore && (
<FactualConsistencyBadge score={activeMessage.factualConsistencyScore} />
)
}
onRetry={
hasError ? () => sendMessage({ query: activeMessage.question, isRetry: true }) : undefined
}
Expand Down Expand Up @@ -241,4 +255,6 @@ export const ChatView = ({
{title}
</button>
);

return <VuiContextProvider>{content}</VuiContextProvider>;
};
47 changes: 47 additions & 0 deletions src/components/FactualConsistencyBadge.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { VuiBadge, VuiFlexContainer, VuiLink, VuiSpinner, VuiText } from "../vui";

interface Props {
score?: number;
}

export const FactualConsistencyBadge = ({ score }: Props) => {
let badge;

if (score === undefined) {
badge = <VuiBadge color="accent">Calculating Factual Consistency Score…</VuiBadge>;
} else {
const sanitizedScore = parseFloat(score.toFixed(2));
let badgeColor = "neutral";

if (sanitizedScore === 0) {
badgeColor = "danger";
} else if (sanitizedScore === 1) {
badgeColor = "success";
}

badge = (
<VuiBadge color={badgeColor as "neutral" | "success" | "danger"}>
Factual Consistency Score: {sanitizedScore}
</VuiBadge>
);
}

return (
<VuiFlexContainer alignItems="center" data-testid="factualConsistencyBadge">
{score === undefined && <VuiSpinner size="s" />}

{badge}

<VuiText size="xs">
<p>
<VuiLink
href="https://docs.vectara.com/docs/api-reference/search-apis/search?#factual-consistency-score"
target="_blank"
>
What's this?
</VuiLink>
</p>
</VuiText>
</VuiFlexContainer>
);
};
1 change: 1 addition & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,4 +93,5 @@ export type ChatTurn = {
question: string;
answer: string;
results: DeserializedSearchResult[];
factualConsistencyScore?: number;
};
47 changes: 36 additions & 11 deletions src/useChat.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useEffect, useRef, useState } from "react";
import { ChatTurn, SummaryLanguage } from "types";
import { ChatDetail, streamQuery, StreamUpdate } from "@vectara/stream-query-client";
import { ChatDetail, FactualConsistencyDetail, streamQuery, StreamUpdate } from "@vectara/stream-query-client";
import { sendSearchRequest } from "utils/sendSearchRequest";
import { deserializeSearchResponse } from "utils/deserializeSearchResponse";

Expand Down Expand Up @@ -46,6 +46,7 @@ export const useChat = ({

const sendMessage = async ({ query, isRetry = false }: { query: string; isRetry?: boolean }) => {
if (isLoading) return;

if (isRetry) {
setHasError(false);
}
Expand Down Expand Up @@ -121,7 +122,8 @@ export const useChat = ({
id: response.summary[0].chat.turnId,
question: recentQuestion.current,
answer: response?.summary[0].text ?? "",
results: deserializeSearchResponse(response) ?? []
results: deserializeSearchResponse(response) ?? [],
factualConsistencyScore: response.summary[0].factualConsistency?.score
}
]);
setActiveMessage(null);
Expand All @@ -140,7 +142,11 @@ export const useChat = ({
setConversationId(null);
};

const onStreamUpdate = ({ references, details, updatedText, isDone }: StreamUpdate) => {
const onStreamUpdate = (update: StreamUpdate) => {
const { references, details, updatedText, isDone } = update;

const factualConsistencyScore = extractFactualConsistencyScore(update);

if (updatedText) {
setIsStreamingResponse(true);
setIsLoading(false);
Expand All @@ -152,18 +158,26 @@ export const useChat = ({
setConversationId(chatDetail.data.conversationId ?? null);
}

if (isDone) {
setActiveMessage((prev) => ({
id: chatDetail ? chatDetail.data.turnId : "",
question: recentQuestion.current,
answer: updatedText ?? "",
results: [...(prev?.results ?? []), ...(references ?? [])],
factualConsistencyScore
}));

const isFactualConsistencyScoreComplete = enableFactualConsistencyScore
? factualConsistencyScore !== undefined
: true;
const isResponseComplete = isDone && isFactualConsistencyScoreComplete;

if (isResponseComplete) {
setIsStreamingResponse(false);
} else {
setActiveMessage((prev) => ({
id: chatDetail ? chatDetail.data.turnId : "",
question: recentQuestion.current,
answer: updatedText ?? "",
results: [...(prev?.results ?? []), ...(references ?? [])]
}));
}
};

// Handle this in an effect instead of directly in the onStreamUpdate callback
// because onStreamUpdate doesn't have access to the latest state of activeMessage.
useEffect(() => {
if (!isStreamingResponse && activeMessage) {
setMessageHistory([...messageHistory, activeMessage]);
Expand All @@ -181,3 +195,14 @@ export const useChat = ({
hasError
};
};

const extractFactualConsistencyScore = (update: StreamUpdate) => {
const { details } = update;
const factualConsistencyDetail = details?.find((detail) => detail.type === "factualConsistency") as
| FactualConsistencyDetail
| undefined;

if (factualConsistencyDetail) {
return factualConsistencyDetail.data.score;
}
};
2 changes: 2 additions & 0 deletions src/vui/components/_index.scss
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
@import "../styleUtils/index";
@import "accordion/index";
@import "badge/index";
@import "button/index";
@import "flex/index";
@import "link/index";
@import "searchInput/index";
@import "spinner/index";
@import "spacer/index";
Expand Down
50 changes: 50 additions & 0 deletions src/vui/components/badge/Badge.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { MouseEvent } from "react";
import classNames from "classnames";
import { getTrackingProps } from "../../utils/getTrackingProps";
import { useVuiContext } from "../context/Context";
import { LinkProps } from "../link/types";

export const BADGE_COLOR = ["accent", "primary", "danger", "warning", "success", "neutral"] as const;

type Props = {
children: React.ReactNode;
className?: string;
color: (typeof BADGE_COLOR)[number];
onClick?: (event: MouseEvent<HTMLButtonElement>) => void;
href?: LinkProps["href"];
target?: LinkProps["target"];
track?: LinkProps["track"];
};

export const VuiBadge = ({ children, className, color, onClick, href, target, track, ...rest }: Props) => {
const { createLink } = useVuiContext();

const classes = classNames(className, "vuiBadge", `vuiBadge--${color}`, {
"vuiBadge--clickable": onClick ?? href
});

if (onClick) {
return (
<button className={classes} onClick={onClick} {...rest}>
{children}
</button>
);
}

if (href) {
return createLink({
className: classes,
href,
onClick,
children,
target,
...getTrackingProps(track)
});
}

return (
<div className={classes} {...rest}>
{children}
</div>
);
};
64 changes: 64 additions & 0 deletions src/vui/components/badge/_index.scss
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
@use "sass:map";

.vuiBadge {
display: inline-block;
font-size: $fontSizeSmall;
line-height: 1;
padding: $sizeXxs $sizeXs;
border-radius: $sizeS;
font-family: inherit;
white-space: nowrap;
text-decoration: none;
}

.vuiBadge--clickable {
cursor: pointer;
}

// Color
$color: (
accent: (
"color": $colorAccent,
"background-color": transparentize($colorAccent, 0.9),
"border-color": transparentize($colorAccent, 0.9)
),
primary: (
"color": $colorPrimary,
"background-color": transparentize($colorPrimary, 0.9),
"border-color": transparentize($colorPrimary, 0.9)
),
success: (
"color": $colorSuccess,
"background-color": transparentize($colorSuccess, 0.9),
"border-color": transparentize($colorSuccess, 0.9)
),
warning: (
"color": $colorWarning,
"background-color": transparentize($colorWarning, 0.9),
"border-color": transparentize($colorWarning, 0.9)
),
danger: (
"color": $colorDanger,
"background-color": transparentize($colorDanger, 0.9),
"border-color": transparentize($colorDanger, 0.9)
),
neutral: (
"color": $colorText,
"background-color": $colorLightShade,
"border-color": transparentize($colorText, 0.9)
)
);

@each $colorName, $colorValue in $color {
.vuiBadge--#{$colorName} {
color: #{map.get($colorValue, "color")} !important;
background-color: #{map.get($colorValue, "background-color")};
border: 1px solid #{map.get($colorValue, "border-color")};
transition: all $transitionSpeed;

&.vuiBadge--clickable:hover {
border-color: #{map.get($colorValue, "color")};
text-decoration: none;
}
}
}
18 changes: 18 additions & 0 deletions src/vui/components/context/Context.test.util.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { render } from "@testing-library/react";
import { Link } from "react-router-dom";
import { LinkProps } from "../link/types";
import { VuiContextProvider } from "./Context";

const linkProvider = (linkConfig: LinkProps) => {
const { className, href, onClick, children, ...rest } = linkConfig;

return (
<Link className={className} to={href ?? ""} onClick={onClick} {...rest}>
{children}
</Link>
);
};

export const renderWithContext = (children: React.ReactNode, ...rest: any) => {
return render(<VuiContextProvider linkProvider={linkProvider}>{children}</VuiContextProvider>, ...rest);
};
Loading

0 comments on commit 415e994

Please sign in to comment.