diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..2c6e5c35 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: Test + +on: + pull_request: + branches: main + +jobs: + integration-tests: + name: Integration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + + - name: Install dependencies + run: npm ci + + - name: Install Playwright dependencies + run: npx playwright install --with-deps + + - name: Run Playwright tests + run: npm test + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: test-results/ diff --git a/.gitignore b/.gitignore index 81f1f4a3..c10d9606 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ node_modules vite.config.js.timestamp-* vite.config.ts.timestamp-* +/test-results /ai/models /sessions/*.json diff --git a/package-lock.json b/package-lock.json index f0b061f1..21aa328c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "tailwind-variants": "^0.2.1" }, "devDependencies": { + "@playwright/test": "^1.43.0", "@sveltejs/adapter-auto": "^3.0.0", "@sveltejs/adapter-cloudflare": "^4.2.1", "@sveltejs/kit": "^2.0.0", @@ -917,6 +918,21 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.43.0.tgz", + "integrity": "sha512-Ebw0+MCqoYflop7wVKj711ccbNlrwTBCtjY5rlbiY9kHL2bCYxq+qltK6uPsVBGGAOb033H2VO0YobcQVxoW7Q==", + "dev": true, + "dependencies": { + "playwright": "1.43.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.25", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.25.tgz", @@ -3544,6 +3560,50 @@ "node": ">= 6" } }, + "node_modules/playwright": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.0.tgz", + "integrity": "sha512-SiOKHbVjTSf6wHuGCbqrEyzlm6qvXcv7mENP+OZon1I07brfZLGdfWV0l/efAzVx7TF3Z45ov1gPEkku9q25YQ==", + "dev": true, + "dependencies": { + "playwright-core": "1.43.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.0.tgz", + "integrity": "sha512-iWFjyBUH97+pUFiyTqSLd8cDMMOS0r2ZYz2qEsPjH8/bX++sbIJT35MSwKnp1r/OQBAqC5XO99xFbJ9XClhf4w==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.4.38", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", diff --git a/package.json b/package.json index cbf8c97d..417bf138 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,14 @@ "dev": "vite dev --host", "build": "vite build", "preview": "vite preview", + "test": "playwright test --trace on", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "prettier --check . && eslint .", "format": "prettier --write ." }, "devDependencies": { + "@playwright/test": "^1.43.0", "@sveltejs/adapter-auto": "^3.0.0", "@sveltejs/adapter-cloudflare": "^4.2.1", "@sveltejs/kit": "^2.0.0", diff --git a/playwright.config.ts b/playwright.config.ts index 1c5d7a1f..0827c2d5 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -6,7 +6,11 @@ const config: PlaywrightTestConfig = { port: 4173 }, testDir: 'tests', - testMatch: /(.+\.)?(test|spec)\.[jt]s/ + testMatch: /(.+\.)?(test|spec)\.[jt]s/, + timeout: 5000, + use: { + trace: 'on-first-retry' + } }; export default config; diff --git a/src/lib/ollama.ts b/src/lib/ollama.ts index 010b0669..c3eed29e 100644 --- a/src/lib/ollama.ts +++ b/src/lib/ollama.ts @@ -2,12 +2,46 @@ import { get } from "svelte/store"; import type { Session } from "$lib/sessions"; import { settingsStore } from "$lib/store"; -interface OllamaCompletionRequest { - model: string; +type OllamaCompletionRequest = { + context: number[]; prompt: string; + model: string; +} + +export type OllamaCompletionResponse = { + model: string; + created_at: string; + response: string; + done: boolean; context: number[]; + total_duration: number; + load_duration: number; + prompt_eval_count: number; + prompt_eval_duration: number; + eval_count: number; + eval_duration: number; } +export type OllamaModel = { + name: string; + model: string; + modified_at: string; + size: number; + digest: string; + details: { + parent_model: string; + format: string; + family: string; + families: string[] | null; + parameter_size: string; + quantization_level: string; + }; +}; + +export type OllamaTagResponse = { + models: OllamaModel[]; +}; + export async function ollamaGenerate(session: Session) { const settings = get(settingsStore); if (!settings) throw new Error('No Ollama server specified'); @@ -23,4 +57,4 @@ export async function ollamaGenerate(session: Session) { headers: { 'Content-Type': 'text/event-stream' }, body: JSON.stringify(payload) }); -} \ No newline at end of file +} diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 412bb8ad..9877ec19 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -2,6 +2,16 @@ import '../app.pcss'; import Button from '$lib/components/ui/button/button.svelte'; import Separator from '$lib/components/ui/separator/separator.svelte'; + import { onMount } from 'svelte'; + + let newSessionId: string; + + function createNewSession() { + // Example: `zbvxte` + newSessionId = Math.random().toString(36).substring(2, 8); + } + + onMount(createNewSession);
@@ -12,7 +22,12 @@
-
diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 07468c3f..55d92852 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -9,9 +9,10 @@ import { slide } from 'svelte/transition'; import Badge from '$lib/components/ui/badge/badge.svelte'; import Separator from '$lib/components/ui/separator/separator.svelte'; + import type { OllamaTagResponse } from '$lib/ollama'; export let ollamaURL: URL | null = null; - let modelList: ModelList | null = null; + let ollamaTagResponse: OllamaTagResponse | null = null; const DETAULT_OLLAMA_SERVER = 'http://localhost:11434'; let ollamaServer = $settingsStore?.ollamaServer || DETAULT_OLLAMA_SERVER; @@ -27,10 +28,6 @@ // key on the Ollama server input. $: typeof ollamaServer === 'string' && getModelsList(); - interface ModelList { - models: Model[]; - } - interface Model { name: string; modified_at: string; @@ -41,11 +38,11 @@ async function getModelsList(): Promise { try { const response = await fetch(`${ollamaServer}/api/tags`); - const data = await response.json(); - modelList = data; + const data = await response.json() as OllamaTagResponse; + ollamaTagResponse = data; serverStatus = 'connected'; } catch { - modelList = null; + ollamaTagResponse = null; serverStatus = 'disconnected'; } } @@ -116,7 +113,6 @@ - {#if ollamaURL && serverStatus === 'disconnected'}

@@ -161,14 +157,14 @@ - {#if modelList} - {#each modelList.models as model} + {#if ollamaTagResponse} + {#each ollamaTagResponse.models as model} {model.name} {/each} {:else} diff --git a/src/routes/[id]/+page.svelte b/src/routes/[id]/+page.svelte index 256dd191..ac16c940 100644 --- a/src/routes/[id]/+page.svelte +++ b/src/routes/[id]/+page.svelte @@ -4,7 +4,7 @@ import Separator from '$lib/components/ui/separator/separator.svelte'; import Textarea from '$lib/components/ui/textarea/textarea.svelte'; - import { ollamaGenerate } from '$lib/ollama'; + import { ollamaGenerate, type OllamaCompletionResponse } from '$lib/ollama'; import { saveSession, type Message, type Session, loadSession } from '$lib/sessions'; import type { PageData } from './$types'; import Article from './Article.svelte'; @@ -77,7 +77,7 @@ const jsonLines = value.split('\n').filter((line) => line); for (const line of jsonLines) { - const { response, context } = JSON.parse(line); + const { response, context } = JSON.parse(line) as OllamaCompletionResponse; completion += response; session.context = context; } @@ -99,10 +99,10 @@

-

+

Session #{session.id}

-

{session.model}

+

{session.model}

diff --git a/src/routes/[id]/Article.svelte b/src/routes/[id]/Article.svelte index c6000725..1f3082c1 100644 --- a/src/routes/[id]/Article.svelte +++ b/src/routes/[id]/Article.svelte @@ -8,7 +8,7 @@
-

{ + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_COMPLETION_RESPONSE_1) + }); + }); + + await sendButton.click(); + await expect(page.locator('article', { hasText: 'I am unable to provide subjective or speculative information, including fight outcomes between individuals.' })).toBeVisible(); + + await promptTextarea.fill("I understand, it's okay"); + await expect(page.locator('article', { hasText: 'No problem! If you have any other questions or would like to discuss something else, feel free to ask' })).not.toBeVisible(); + await expect(page.locator('article', { hasText: "I understand, it's okay" })).not.toBeVisible(); + + await page.route('**/generate', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_COMPLETION_RESPONSE_2) + }); + }); + + await page.keyboard.press('Enter'); + await expect(page.locator('article', { hasText: "I understand, it's okay" })).toBeVisible(); + await expect(page.locator('article', { hasText: 'No problem! If you have any other questions or would like to discuss something else, feel free to ask' })).toBeVisible(); + await expect(page.getByText('AI')).toHaveCount(2); + expect(await page.getByText('You').count()).toBeGreaterThan(1); + + // Check the session is saved to localStorage + await newSessionButton.click(); + await expect(articleLocator).toHaveCount(0); + + await page.goBack(); + await expect(articleLocator).toHaveCount(4); +}); + +test('generates a random session id', async ({ page }) => { + await page.goto('/'); + + const sessionIds = []; + const newSessionButton = page.getByText('New session'); + + // Check it generates a new session id 3 times in a row + for (let i = 0; i < 3; i++) { + const sessionId = await newSessionButton.getAttribute('href'); + expect(sessionId).toMatch(/[a-z0-9]{2,8}/); + sessionIds.push(sessionId); + await newSessionButton.click(); + } + + expect(new Set(sessionIds).size).toBe(3); +}); + +test.skip('handles API error when generating AI response', async ({ page }) => { + // TODO: Implement the test +}); + +test.skip('displays system message when an error occurs', async ({ page }) => { + // TODO: Implement the test +}); + +test.skip('auto-scrolls to the bottom when new messages are added', async ({ page }) => { + // TODO: Implement the test +}); diff --git a/tests/settings.test.ts b/tests/settings.test.ts new file mode 100644 index 00000000..52c67cee --- /dev/null +++ b/tests/settings.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from '@playwright/test'; +import type { OllamaTagResponse } from '$lib/ollama'; +import { MOCK_API_TAGS_RESPONSE } from './utils'; + +test.beforeEach(async ({ page }) => { + // Enable request interception + await page.route('**/api/tags', async (route) => { + // Fulfill the request with the mock response + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_API_TAGS_RESPONSE) + }); + }); +}); + +test('displays model list and updates settings store', async ({ page }) => { + await page.goto('/'); + + // Wait for the model list to be loaded + await page.waitForSelector('label:has-text("Model")'); + + // Click on the button that opens the model list + await page.click('button[data-melt-select-trigger]'); + + // Wait for the model list to be visible + await page.waitForSelector('div[role="listbox"]'); + + // Check if the model list contains the expected models + const modelNames = await page.$$eval('div[role="option"]', options => + options.map(option => option.textContent?.trim()) + ); + expect(modelNames).toContain('gemma:7b'); + expect(modelNames).toContain('mistral:latest'); + + // Select a model by clicking on its name + await page.click('div[role="option"]:has-text("mistral:latest")'); + + // Check if the settings store is updated with the selected model + const localStorageValue = await page.evaluate(() => window.localStorage.getItem('hollama-settings')); + expect(localStorageValue).toContain('"ollamaModel":"mistral:latest"'); +}); + +test('handles server status updates correctly', async ({ page }) => { + await page.goto('/'); + await page.waitForSelector('label:has-text("Server")'); + + // Mock the API to return a successful response + await page.route('**/api/tags', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_API_TAGS_RESPONSE) + }); + }); + + // Trigger a new API request by typing in the input field + await page.fill('input[placeholder="http://localhost:11434"]', 'http://example.com'); + + // Wait for the server status to be updated to "connected" + await expect(page.getByText('connected', { exact: true })).toBeVisible(); + await expect(page.getByText('connected', { exact: true })).toHaveClass(/bg-emerald-600/); + + // Mock the API to return an error response + await page.route('**/api/tags', async (route) => { + await route.abort(); + }); + + // Trigger a new API request by typing in the input field + await page.fill('input[placeholder="http://localhost:11434"]', 'http://example.com/invalid'); + + // Wait for the server status to be updated to "disconnected" + await expect(page.getByText('disconnected')).toBeVisible(); + await expect(page.getByText('disconnected')).toHaveClass(/bg-amber-600/); +}); diff --git a/tests/test.ts b/tests/test.ts deleted file mode 100644 index 5816be41..00000000 --- a/tests/test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { expect, test } from '@playwright/test'; - -test('index page has expected h1', async ({ page }) => { - await page.goto('/'); - await expect(page.getByRole('heading', { name: 'Welcome to SvelteKit' })).toBeVisible(); -}); diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 00000000..423618dd --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,183 @@ +import type { OllamaCompletionResponse, OllamaTagResponse } from "$lib/ollama"; + +export const MOCK_API_TAGS_RESPONSE: OllamaTagResponse = { + models: [ + { + name: 'gemma:7b', + model: 'gemma:7b', + modified_at: '2024-04-08T21:41:35.217983842-04:00"', + size: 5011853225, + digest: 'a72c7f4d0a15522df81486d13ce432c79e191bda2558d024fbad4362c4f7cbf8', + details: { + parent_model: '', + format: 'gguf', + family: 'gemma', + families: ["gemma"], + parameter_size: '9B', + quantization_level: 'Q4_0' + } + }, + { + name: "mistral:latest", + model: "mistral:latest", + modified_at: "2023-11-24T16:32:44.035655802-05:00", + size: 4108916866, + digest: "d364aa8d131ef7abfc1275db682d281a307d9451fc00f96abe154d0059b0be49", + details: { + parent_model: "", + format: "gguf", + family: "llama", + families: null, + parameter_size: "7B", + quantization_level: "Q4_0" + } + } + ] +}; + +export const MOCK_COMPLETION_RESPONSE_1: OllamaCompletionResponse = { + model: "gemma:7b", + created_at: "2024-04-10T22:54:40.310905Z", + response: "I am unable to provide subjective or speculative information, including fight outcomes between individuals.", + done: true, + context: [ + 106, + 1645, + 108, + 6571, + 1134, + 3709, + 575, + 476, + 5900, + 1865, + 29349, + 29678, + 578, + 36171, + 68187, + 235336, + 107, + 108, + 106, + 2516, + 108, + 235285, + 1144, + 14321, + 577, + 3658, + 45927, + 689, + 85979, + 2113, + 235269, + 3359, + 5900, + 18701, + 1865, + 9278, + 235265, + 107, + 108 + ], + total_duration: 564546083, + load_duration: 419166, + prompt_eval_count: 18, + prompt_eval_duration: 267210000, + eval_count: 17, + eval_duration: 296374000, +}; + +export const MOCK_COMPLETION_RESPONSE_2: OllamaCompletionResponse = { + model: "gemma:7b", + created_at: "2024-04-10T23:08:33.419483Z", + response: "No problem! If you have any other questions or would like to discuss something else, feel free to ask.", + done: true, + context: [ + 106, + 1645, + 108, + 6571, + 1134, + 3709, + 575, + 476, + 5900, + 1865, + 29349, + 29678, + 578, + 36171, + 68187, + 235336, + 107, + 108, + 106, + 2516, + 108, + 235285, + 1144, + 14321, + 577, + 3658, + 45927, + 689, + 85979, + 2113, + 235269, + 3359, + 5900, + 18701, + 1865, + 9278, + 235265, + 107, + 108, + 106, + 1645, + 108, + 235285, + 3508, + 235269, + 665, + 235303, + 235256, + 12763, + 107, + 108, + 106, + 2516, + 108, + 1294, + 3210, + 235341, + 1927, + 692, + 791, + 1089, + 1156, + 3920, + 689, + 1134, + 1154, + 577, + 9742, + 2775, + 1354, + 235269, + 2375, + 2223, + 577, + 5291, + 235265, + 107, + 108 + ], + total_duration: 1574338000, + load_duration: 1044484792, + prompt_eval_count: 55, + prompt_eval_duration: 130165000, + eval_count: 23, + eval_duration: 399362000, +};