test(agent): add golden test. (#684)

* test(agent): add golden test.

* test(agent): update golden test.

* test(agent): remove docker-compose.

* test(agent): update golden test.

clients/tabby-agent/.prettierignore

@@ -0,0 +1 @@
+tests/golden/**

clients/tabby-agent/package.json

@@ -12,6 +12,7 @@
     "build": "tsup",
     "test": "mocha",
     "test:watch": "env TEST_LOG_DEBUG=1 mocha --watch",
+    "test:golden": "mocha --grep golden ./tests/golden.test.ts",
     "lint": "prettier --write .",
     "lint:check": "prettier --check ."
   },

clients/tabby-agent/src/TabbyAgent.ts

@@ -104,15 +104,18 @@ export class TabbyAgent extends EventEmitter implements Agent {
       // If auth token is provided, use it directly.
       this.auth = null;
     }
-    await this.setupApi();
 
     // If server config changed, clear server related state
     if (!deepEqual(oldConfig.server, this.config.server)) {
       this.serverHealthState = null;
       this.completionProviderStats.resetWindowed();
       this.popIssue("slowCompletionResponseTime");
       this.popIssue("highCompletionTimeoutRate");
+    }
 
+    await this.setupApi();
+
+    if (!deepEqual(oldConfig.server, this.config.server)) {
       // If server config changed and status remain `unauthorized`, we want to emit `authRequired` again.
       // but `changeStatus` will not emit `authRequired` if status is not changed, so we emit it manually here.
       if (oldStatus === "unauthorized" && this.status === "unauthorized") {

clients/tabby-agent/tests/golden.test.ts

@@ -0,0 +1,132 @@
+// This golden test requires local Tabby server to be running on port 8087.
+// The server should use tabby linux image version 0.4.0, with model TabbyML/StarCoder-1B,
+// cuda backend, and without any repository specified for RAG.
+
+import { spawn } from "child_process";
+import readline from "readline";
+import path from "path";
+import * as fs from "fs-extra";
+import { expect } from "chai";
+
+describe("agent golden test", () => {
+  const agent = spawn("node", [path.join(__dirname, "../dist/cli.js")]);
+  const output: any[] = [];
+  readline.createInterface({ input: agent.stdout }).on("line", (line) => {
+    output.push(JSON.parse(line));
+  });
+
+  const waitForResponse = (requestId, timeout = 1000) => {
+    return new Promise<void>((resolve, reject) => {
+      const start = Date.now();
+      const interval = setInterval(() => {
+        if (output.find((item) => item[0] === requestId)) {
+          clearInterval(interval);
+          resolve();
+        } else if (Date.now() - start > timeout) {
+          clearInterval(interval);
+          reject(new Error("Timeout"));
+        }
+      }, 10);
+    });
+  };
+
+  const createGoldenTest = async (goldenFilepath) => {
+    const content = await fs.readFile(goldenFilepath, "utf8");
+    const language = path.basename(goldenFilepath, path.extname(goldenFilepath)).replace(/^\d+-/g, "");
+    const replaceStart = content.indexOf("⏩");
+    const insertStart = content.indexOf("⏭");
+    const insertEnd = content.indexOf("⏮");
+    const replaceEnd = content.indexOf("⏪");
+    const prefix = content.slice(0, replaceStart);
+    const replacePrefix = content.slice(replaceStart + 1, insertStart);
+    const suggestion = content.slice(insertStart + 1, insertEnd);
+    const replaceSuffix = content.slice(insertEnd + 1, replaceEnd);
+    const suffix = content.slice(replaceEnd + 1);
+    const request = {
+      filepath: goldenFilepath,
+      language,
+      text: prefix + replacePrefix + replaceSuffix + suffix,
+      position: prefix.length + replacePrefix.length,
+      manually: true,
+    };
+    const text = replacePrefix + suggestion;
+    const expected =
+      text.length == 0
+        ? { choices: [] }
+        : {
+            choices: [
+              {
+                index: 0,
+                text,
+                replaceRange: {
+                  start: prefix.length,
+                  end: prefix.length + replacePrefix.length + replaceSuffix.length,
+                },
+              },
+            ],
+          };
+    return { request, expected };
+  };
+
+  it("initialize", async () => {
+    const requestId = 1;
+    const config = {
+      server: {
+        endpoint: "http://127.0.0.1:8087",
+        token: "",
+        requestHeaders: {},
+        requestTimeout: 30000,
+      },
+      completion: {
+        prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 },
+        debounce: { mode: "adaptive", interval: 250 },
+        timeout: { auto: 4000, manually: 4000 },
+      },
+      postprocess: {
+        limitScope: {
+          experimentalSyntax: false,
+          indentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
+        },
+      },
+      logs: { level: "debug" },
+      anonymousUsageTracking: { disable: true },
+    };
+    const initRequest = [
+      requestId,
+      {
+        func: "initialize",
+        args: [{ config }],
+      },
+    ];
+
+    agent.stdin.write(JSON.stringify(initRequest) + "\n");
+    await waitForResponse(requestId);
+    expect(output.shift()).to.deep.equal([0, { event: "statusChanged", status: "ready" }]);
+    expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config }]);
+    expect(output.shift()).to.deep.equal([requestId, true]);
+  });
+
+  const goldenFiles = fs.readdirSync(path.join(__dirname, "golden")).map((file) => {
+    return {
+      name: file,
+      path: path.join(__dirname, "golden", file),
+    };
+  });
+  const baseRequestId = 2;
+  goldenFiles.forEach((goldenFile, index) => {
+    it(goldenFile.name, async () => {
+      const test = await createGoldenTest(goldenFile.path);
+      const requestId = baseRequestId + index;
+      const request = [requestId, { func: "provideCompletions", args: [test.request] }];
+      agent.stdin.write(JSON.stringify(request) + "\n");
+      await waitForResponse(requestId);
+      const response = output.shift();
+      expect(response[0]).to.equal(requestId);
+      expect(response[1].choices).to.deep.equal(test.expected.choices);
+    });
+  });
+
+  after(() => {
+    agent.kill();
+  });
+});

clients/tabby-agent/tests/golden/0-python.py

@@ -0,0 +1,6 @@
+def fib(n):
+    ⏩⏭    if n == 0:
+            return 0
+        if n == 1:
+            return 1⏮⏪
+        return fib(n - 1) + fib(n - 2)

clients/tabby-agent/tests/golden/1-python.py

@@ -0,0 +1,17 @@
+import datetime
+
+def parse_expenses(expenses_string):
+    """Parse the list of expenses and return the list of triples (date, value, currency).
+    Ignore lines starting with #.
+    Parse the date using datetime.
+    Example expenses_string:
+        2016-01-02 -34.01 USD
+        2016-01-03 2.59 DKK
+        2016-01-03 -2.72 EUR
+    """
+    for line in expenses_string.split('\\n'):
+        ⏩⏭if line.startswith('#'):
+            continue
+        date, value, currency = line.split(' - ')
+        date = datetime.datetime.strptime(date, '%Y-%m-%d')
+        yield date, float(value), currency⏮⏪

clients/tabby-agent/tests/golden/2-typescript.ts

@@ -0,0 +1,11 @@
+export class Foo {
+  private _foo: number;
+
+  constructor() {
+    this._foo = 1;
+  }
+
+  update(value): Foo {
+    this._foo = max(⏩⏭this._foo, value⏮⏪)
+  }
+}

clients/tabby-agent/tests/golden/3-typescript.ts

@@ -0,0 +1 @@
+function fibonacci(⏩⏭⏮⏪)

clients/tabby-agent/tests/golden/4-typescript.ts

@@ -0,0 +1,9 @@
+function fibonacci(⏩⏭n) {
+  if (n === 0) {
+    return 0;
+  }
+  if (n === 1) {
+    return 1;
+  }
+  return fibonacci(n - 1) + fibonacci(n - 2);
+}⏮⏪

clients/tabby-agent/tests/golden/5-go.go

@@ -0,0 +1,9 @@
+func fibonacci(⏩⏭n int) int {
+	if n == 0 {
+		return 0
+	}
+	if n == 1 {
+		return 1
+	}
+	return fibonacci(n-1) + fibonacci(n-2)
+}⏮⏪

clients/tabby-agent/tests/golden/6-rust.rs

@@ -0,0 +1,9 @@
+fn fibonacci(⏩⏭n: number): number {
+  if (n === 0) {
+    return 0;
+  }
+  if (n === 1) {
+    return 1;
+  }
+  return fibonacci(n - 1) + fibonacci(n - 2);
+}⏮⏪

clients/tabby-agent/tests/golden/7-ruby.rb

@@ -0,0 +1,4 @@
+def fibonacci(n)
+  return n if n <= 1
+  ⏩⏭fibonacci(n - 1) + fibonacci(n - 2)⏮⏪
+end