From 1fb7cc53fba3a6733294d1a04616cf0c3a102345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Sun, 27 Oct 2024 13:08:56 +0100 Subject: [PATCH 01/15] chore: Add unit tests for GoogleVertexAICompletonService --- ...google-vertexai-completion-service.spec.ts | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 packages/navie/test/services/google-vertexai-completion-service.spec.ts diff --git a/packages/navie/test/services/google-vertexai-completion-service.spec.ts b/packages/navie/test/services/google-vertexai-completion-service.spec.ts new file mode 100644 index 0000000000..0d45fc9d21 --- /dev/null +++ b/packages/navie/test/services/google-vertexai-completion-service.spec.ts @@ -0,0 +1,97 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { ChatVertexAI } from '@langchain/google-vertexai-web'; + +import Trajectory from '../../src/lib/trajectory'; +import GoogleVertexAICompletionService from '../../src/services/google-vertexai-completion-service'; +import { z } from 'zod'; +import Message from '../../src/message'; + +jest.mock('@langchain/google-vertexai-web'); + +describe('GoogleVertexAICompletionService', () => { + let service: GoogleVertexAICompletionService; + let trajectory: Trajectory; + const modelName = 'the-model-name'; + const temperature = 0.2; + + beforeEach(() => { + trajectory = new Trajectory(); + service = new GoogleVertexAICompletionService(modelName, temperature, trajectory); + }); + + const ChatVertexAIMock = jest.mocked(ChatVertexAI); + const responseMock = jest.fn(); + function mockCompletion(response: string) { + responseMock.mockReturnValue(response); + } + + beforeEach(() => + ChatVertexAIMock.mockImplementation( + () => + ({ + // eslint-disable-next-line @typescript-eslint/require-await + stream: jest.fn(async function* () { + yield { content: responseMock() }; + }), + invoke() { + return { content: responseMock() }; + }, + } as never) + ) + ); + afterEach(jest.resetAllMocks); + + describe('complete', () => { + it('returns the parsed completion', async () => { + mockCompletion('hello world'); + const messages: Message[] = []; + + expect(await collect(service.complete(messages))).toEqual(['hello world']); + }); + + it('retries on errors', async () => { + jest.useFakeTimers(); + mockCompletion('hello world'); + responseMock.mockImplementationOnce(() => { + throw new Error('error'); + }); + const messages: Message[] = []; + + // eslint-disable-next-line jest/valid-expect + const result = expect(collect(service.complete(messages))).resolves.toEqual(['hello world']); + await jest.advanceTimersByTimeAsync(1000); + jest.useRealTimers(); + return result; + }); + }); + + describe('json', () => { + it('returns the parsed JSON', async () => { + mockCompletion('{"foo": "bar"}'); + const schema = z.object({ foo: z.string() }); + const messages: Message[] = []; + const result = await service.json(messages, schema); + expect(result).toEqual({ foo: 'bar' }); + }); + }); + + describe('miniModelName', () => { + it('returns the default mini model name', () => { + expect(service.miniModelName).toEqual('gemini-1.5-flash-002'); + }); + + it('overrides the mini model name with the environment variable', () => { + const miniModelName = 'the-mini-model-name'; + process.env.APPMAP_NAVIE_MINI_MODEL = miniModelName; + expect(service.miniModelName).toEqual(miniModelName); + }); + }); +}); + +async function collect(generator: AsyncIterable): Promise { + const chunks: string[] = []; + for await (const chunk of generator) { + chunks.push(chunk); + } + return chunks; +} From 7adbba361ada7e392f262ab63dc271de6b163f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Sun, 27 Oct 2024 20:58:31 +0100 Subject: [PATCH 02/15] chore: Update jest in navie, dedupe --- packages/navie/package.json | 4 +- yarn.lock | 663 +----------------------------------- 2 files changed, 15 insertions(+), 652 deletions(-) diff --git a/packages/navie/package.json b/packages/navie/package.json index 508db7eccd..3c7c98ed3b 100644 --- a/packages/navie/package.json +++ b/packages/navie/package.json @@ -20,7 +20,7 @@ "license": "MIT + Commons Clause", "devDependencies": { "@tsconfig/node-lts": "^20.1.3", - "@types/jest": "^29.4.1", + "@types/jest": "^29.5.14", "@types/node": "20.1.0", "@typescript-eslint/eslint-plugin": "^8.6.0", "@typescript-eslint/parser": "^8.6.0", @@ -32,7 +32,7 @@ "eslint-plugin-jest": "^28.8.3", "eslint-plugin-promise": "^6.0.1", "eslint-plugin-unicorn": "^39.0.0", - "jest": "^29.5.0", + "jest": "^29.7.0", "ts-jest": "^29.2.5", "ts-node": "^10.4.0", "tsc": "^2.0.3", diff --git a/yarn.lock b/yarn.lock index 2c3938d5b4..531892071d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -460,7 +460,7 @@ __metadata: "@langchain/google-vertexai-web": ^0.1.0 "@langchain/openai": ^0.2.7 "@tsconfig/node-lts": ^20.1.3 - "@types/jest": ^29.4.1 + "@types/jest": ^29.5.14 "@types/node": 20.1.0 "@typescript-eslint/eslint-plugin": ^8.6.0 "@typescript-eslint/parser": ^8.6.0 @@ -473,7 +473,7 @@ __metadata: eslint-plugin-promise: ^6.0.1 eslint-plugin-unicorn: ^39.0.0 fast-xml-parser: ^4.4.0 - jest: ^29.5.0 + jest: ^29.7.0 js-yaml: ^4.1.0 jsdom: ^16.6.0 langchain: ^0.2.16 @@ -6028,20 +6028,6 @@ __metadata: languageName: node linkType: hard -"@jest/console@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/console@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - "@types/node": "*" - chalk: ^4.0.0 - jest-message-util: ^29.5.0 - jest-util: ^29.5.0 - slash: ^3.0.0 - checksum: 9f4f4b8fabd1221361b7f2e92d4a90f5f8c2e2b29077249996ab3c8b7f765175ffee795368f8d6b5b2bb3adb32dc09319f7270c7c787b0d259e624e00e0f64a5 - languageName: node - linkType: hard - "@jest/console@npm:^29.7.0": version: 29.7.0 resolution: "@jest/console@npm:29.7.0" @@ -6133,47 +6119,6 @@ __metadata: languageName: node linkType: hard -"@jest/core@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/core@npm:29.5.0" - dependencies: - "@jest/console": ^29.5.0 - "@jest/reporters": ^29.5.0 - "@jest/test-result": ^29.5.0 - "@jest/transform": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - ansi-escapes: ^4.2.1 - chalk: ^4.0.0 - ci-info: ^3.2.0 - exit: ^0.1.2 - graceful-fs: ^4.2.9 - jest-changed-files: ^29.5.0 - jest-config: ^29.5.0 - jest-haste-map: ^29.5.0 - jest-message-util: ^29.5.0 - jest-regex-util: ^29.4.3 - jest-resolve: ^29.5.0 - jest-resolve-dependencies: ^29.5.0 - jest-runner: ^29.5.0 - jest-runtime: ^29.5.0 - jest-snapshot: ^29.5.0 - jest-util: ^29.5.0 - jest-validate: ^29.5.0 - jest-watcher: ^29.5.0 - micromatch: ^4.0.4 - pretty-format: ^29.5.0 - slash: ^3.0.0 - strip-ansi: ^6.0.0 - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - checksum: 9e8f5243fe82d5a57f3971e1b96f320058df7c315328a3a827263f3b17f64be10c80f4a9c1b1773628b64d2de6d607c70b5b2d5bf13e7f5ad04223e9ef6aac06 - languageName: node - linkType: hard - "@jest/core@npm:^29.7.0": version: 29.7.0 resolution: "@jest/core@npm:29.7.0" @@ -6239,18 +6184,6 @@ __metadata: languageName: node linkType: hard -"@jest/environment@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/environment@npm:29.5.0" - dependencies: - "@jest/fake-timers": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - jest-mock: ^29.5.0 - checksum: 921de6325cd4817dec6685e5ff299b499b6379f3f9cf489b4b13588ee1f3820a0c77b49e6a087996b6de8f629f6f5251e636cba08d1bdb97d8071cc7d033c88a - languageName: node - linkType: hard - "@jest/environment@npm:^29.7.0": version: 29.7.0 resolution: "@jest/environment@npm:29.7.0" @@ -6281,16 +6214,6 @@ __metadata: languageName: node linkType: hard -"@jest/expect@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/expect@npm:29.5.0" - dependencies: - expect: ^29.5.0 - jest-snapshot: ^29.5.0 - checksum: bd10e295111547e6339137107d83986ab48d46561525393834d7d2d8b2ae9d5626653f3f5e48e5c3fa742ac982e97bdf1f541b53b9e1d117a247b08e938527f6 - languageName: node - linkType: hard - "@jest/expect@npm:^29.7.0": version: 29.7.0 resolution: "@jest/expect@npm:29.7.0" @@ -6326,20 +6249,6 @@ __metadata: languageName: node linkType: hard -"@jest/fake-timers@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/fake-timers@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - "@sinonjs/fake-timers": ^10.0.2 - "@types/node": "*" - jest-message-util: ^29.5.0 - jest-mock: ^29.5.0 - jest-util: ^29.5.0 - checksum: 69930c6922341f244151ec0d27640852ec96237f730fc024da1f53143d31b43cde75d92f9d8e5937981cdce3b31416abc3a7090a0d22c2377512c4a6613244ee - languageName: node - linkType: hard - "@jest/fake-timers@npm:^29.7.0": version: 29.7.0 resolution: "@jest/fake-timers@npm:29.7.0" @@ -6365,18 +6274,6 @@ __metadata: languageName: node linkType: hard -"@jest/globals@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/globals@npm:29.5.0" - dependencies: - "@jest/environment": ^29.5.0 - "@jest/expect": ^29.5.0 - "@jest/types": ^29.5.0 - jest-mock: ^29.5.0 - checksum: b309ab8f21b571a7c672608682e84bbdd3d2b554ddf81e4e32617fec0a69094a290ab42e3c8b2c66ba891882bfb1b8b2736720ea1285b3ad646d55c2abefedd9 - languageName: node - linkType: hard - "@jest/globals@npm:^29.7.0": version: 29.7.0 resolution: "@jest/globals@npm:29.7.0" @@ -6456,43 +6353,6 @@ __metadata: languageName: node linkType: hard -"@jest/reporters@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/reporters@npm:29.5.0" - dependencies: - "@bcoe/v8-coverage": ^0.2.3 - "@jest/console": ^29.5.0 - "@jest/test-result": ^29.5.0 - "@jest/transform": ^29.5.0 - "@jest/types": ^29.5.0 - "@jridgewell/trace-mapping": ^0.3.15 - "@types/node": "*" - chalk: ^4.0.0 - collect-v8-coverage: ^1.0.0 - exit: ^0.1.2 - glob: ^7.1.3 - graceful-fs: ^4.2.9 - istanbul-lib-coverage: ^3.0.0 - istanbul-lib-instrument: ^5.1.0 - istanbul-lib-report: ^3.0.0 - istanbul-lib-source-maps: ^4.0.0 - istanbul-reports: ^3.1.3 - jest-message-util: ^29.5.0 - jest-util: ^29.5.0 - jest-worker: ^29.5.0 - slash: ^3.0.0 - string-length: ^4.0.1 - strip-ansi: ^6.0.0 - v8-to-istanbul: ^9.0.1 - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - checksum: 481268aac9a4a75cc49c4df1273d6b111808dec815e9d009dad717c32383ebb0cebac76e820ad1ab44e207540e1c2fe1e640d44c4f262de92ab1933e057fdeeb - languageName: node - linkType: hard - "@jest/reporters@npm:^29.7.0": version: 29.7.0 resolution: "@jest/reporters@npm:29.7.0" @@ -6570,17 +6430,6 @@ __metadata: languageName: node linkType: hard -"@jest/source-map@npm:^29.4.3": - version: 29.4.3 - resolution: "@jest/source-map@npm:29.4.3" - dependencies: - "@jridgewell/trace-mapping": ^0.3.15 - callsites: ^3.0.0 - graceful-fs: ^4.2.9 - checksum: 2301d225145f8123540c0be073f35a80fd26a2f5e59550fd68525d8cea580fb896d12bf65106591ffb7366a8a19790076dbebc70e0f5e6ceb51f81827ed1f89c - languageName: node - linkType: hard - "@jest/source-map@npm:^29.6.3": version: 29.6.3 resolution: "@jest/source-map@npm:29.6.3" @@ -6615,18 +6464,6 @@ __metadata: languageName: node linkType: hard -"@jest/test-result@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/test-result@npm:29.5.0" - dependencies: - "@jest/console": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/istanbul-lib-coverage": ^2.0.0 - collect-v8-coverage: ^1.0.0 - checksum: 2e8ff5242227ab960c520c3ea0f6544c595cc1c42fa3873c158e9f4f685f4ec9670ec08a4af94ae3885c0005a43550a9595191ffbc27a0965df27d9d98bbf901 - languageName: node - linkType: hard - "@jest/test-result@npm:^29.7.0": version: 29.7.0 resolution: "@jest/test-result@npm:29.7.0" @@ -6663,18 +6500,6 @@ __metadata: languageName: node linkType: hard -"@jest/test-sequencer@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/test-sequencer@npm:29.5.0" - dependencies: - "@jest/test-result": ^29.5.0 - graceful-fs: ^4.2.9 - jest-haste-map: ^29.5.0 - slash: ^3.0.0 - checksum: eca34b4aeb2fda6dfb7f9f4b064c858a7adf64ec5c6091b6f4ed9d3c19549177cbadcf1c615c4c182688fa1cf085c8c55c3ca6eea40719a34554b0bf071d842e - languageName: node - linkType: hard - "@jest/test-sequencer@npm:^29.7.0": version: 29.7.0 resolution: "@jest/test-sequencer@npm:29.7.0" @@ -6757,29 +6582,6 @@ __metadata: languageName: node linkType: hard -"@jest/transform@npm:^29.5.0": - version: 29.5.0 - resolution: "@jest/transform@npm:29.5.0" - dependencies: - "@babel/core": ^7.11.6 - "@jest/types": ^29.5.0 - "@jridgewell/trace-mapping": ^0.3.15 - babel-plugin-istanbul: ^6.1.1 - chalk: ^4.0.0 - convert-source-map: ^2.0.0 - fast-json-stable-stringify: ^2.1.0 - graceful-fs: ^4.2.9 - jest-haste-map: ^29.5.0 - jest-regex-util: ^29.4.3 - jest-util: ^29.5.0 - micromatch: ^4.0.4 - pirates: ^4.0.4 - slash: ^3.0.0 - write-file-atomic: ^4.0.2 - checksum: d55d604085c157cf5112e165ff5ac1fa788873b3b31265fb4734ca59892ee24e44119964cc47eb6d178dd9512bbb6c576d1e20e51a201ff4e24d31e818a1c92d - languageName: node - linkType: hard - "@jest/types@npm:^24.3.0, @jest/types@npm:^24.9.0": version: 24.9.0 resolution: "@jest/types@npm:24.9.0" @@ -6945,7 +6747,7 @@ __metadata: languageName: node linkType: hard -"@jridgewell/trace-mapping@npm:^0.3.12, @jridgewell/trace-mapping@npm:^0.3.14, @jridgewell/trace-mapping@npm:^0.3.15, @jridgewell/trace-mapping@npm:^0.3.17": +"@jridgewell/trace-mapping@npm:^0.3.12, @jridgewell/trace-mapping@npm:^0.3.14, @jridgewell/trace-mapping@npm:^0.3.17": version: 0.3.17 resolution: "@jridgewell/trace-mapping@npm:0.3.17" dependencies: @@ -11148,13 +10950,13 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:*": - version: 27.4.0 - resolution: "@types/jest@npm:27.4.0" +"@types/jest@npm:*, @types/jest@npm:^29.4.1, @types/jest@npm:^29.5.14, @types/jest@npm:^29.5.2, @types/jest@npm:^29.5.4": + version: 29.5.14 + resolution: "@types/jest@npm:29.5.14" dependencies: - jest-diff: ^27.0.0 - pretty-format: ^27.0.0 - checksum: d2350267f954f9a2e4a15e5f02fbf19a77abfb9fd9e57a954de1fb0e9a0d3d5f8d3646ac7d9c42aeb4b4d828d2e70624ec149c85bb50a48634a54eed8429e1f8 + expect: ^29.0.0 + pretty-format: ^29.0.0 + checksum: 18dba4623f26661641d757c63da2db45e9524c9be96a29ef713c703a9a53792df9ecee9f7365a0858ddbd6440d98fe6b65ca67895ca5884b73cbc7ffc11f3838 languageName: node linkType: hard @@ -11167,36 +10969,6 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:^29.4.1": - version: 29.4.1 - resolution: "@types/jest@npm:29.4.1" - dependencies: - expect: ^29.0.0 - pretty-format: ^29.0.0 - checksum: 75cdd1804615b663b88844eef61cb7c8104d65baf718190a1cd88ce199412dbe2ee136df3698604cc258a9a62ba7cbd7bd7712a17d8cb748909263ef56d6aac3 - languageName: node - linkType: hard - -"@types/jest@npm:^29.5.2": - version: 29.5.2 - resolution: "@types/jest@npm:29.5.2" - dependencies: - expect: ^29.0.0 - pretty-format: ^29.0.0 - checksum: 7d205599ea3cccc262bad5cc173d3242d6bf8138c99458509230e4ecef07a52d6ddcde5a1dbd49ace655c0af51d2dbadef3748697292ea4d86da19d9e03e19c0 - languageName: node - linkType: hard - -"@types/jest@npm:^29.5.4": - version: 29.5.4 - resolution: "@types/jest@npm:29.5.4" - dependencies: - expect: ^29.0.0 - pretty-format: ^29.0.0 - checksum: 38ed5942f44336452efd0f071eab60aaa57cd8d46530348d0a3aa5a691dcbf1366c4ca8f6ee8364efb45b4413bfefae443e5d4f469246a472a03b21ac11cd4ed - languageName: node - linkType: hard - "@types/js-yaml@npm:^4.0.3, @types/js-yaml@npm:^4.0.5": version: 4.0.5 resolution: "@types/js-yaml@npm:4.0.5" @@ -15267,23 +15039,6 @@ __metadata: languageName: node linkType: hard -"babel-jest@npm:^29.5.0": - version: 29.5.0 - resolution: "babel-jest@npm:29.5.0" - dependencies: - "@jest/transform": ^29.5.0 - "@types/babel__core": ^7.1.14 - babel-plugin-istanbul: ^6.1.1 - babel-preset-jest: ^29.5.0 - chalk: ^4.0.0 - graceful-fs: ^4.2.9 - slash: ^3.0.0 - peerDependencies: - "@babel/core": ^7.8.0 - checksum: eafb6d37deb71f0c80bf3c80215aa46732153e5e8bcd73f6ff47d92e5c0c98c8f7f75995d0efec6289c371edad3693cd8fa2367b0661c4deb71a3a7117267ede - languageName: node - linkType: hard - "babel-jest@npm:^29.7.0": version: 29.7.0 resolution: "babel-jest@npm:29.7.0" @@ -15393,18 +15148,6 @@ __metadata: languageName: node linkType: hard -"babel-plugin-jest-hoist@npm:^29.5.0": - version: 29.5.0 - resolution: "babel-plugin-jest-hoist@npm:29.5.0" - dependencies: - "@babel/template": ^7.3.3 - "@babel/types": ^7.3.3 - "@types/babel__core": ^7.1.14 - "@types/babel__traverse": ^7.0.6 - checksum: 099b5254073b6bc985b6d2d045ad26fb8ed30ff8ae6404c4fe8ee7cd0e98a820f69e3dfb871c7c65aae0f4b65af77046244c07bb92d49ef9005c90eedf681539 - languageName: node - linkType: hard - "babel-plugin-jest-hoist@npm:^29.6.3": version: 29.6.3 resolution: "babel-plugin-jest-hoist@npm:29.6.3" @@ -15647,18 +15390,6 @@ __metadata: languageName: node linkType: hard -"babel-preset-jest@npm:^29.5.0": - version: 29.5.0 - resolution: "babel-preset-jest@npm:29.5.0" - dependencies: - babel-plugin-jest-hoist: ^29.5.0 - babel-preset-current-node-syntax: ^1.0.0 - peerDependencies: - "@babel/core": ^7.0.0 - checksum: 5566ca2762766c9319b4973d018d2fa08c0fcf6415c72cc54f4c8e7199e851ea8f5e6c6730f03ed7ed44fc8beefa959dd15911f2647dee47c615ff4faeddb1ad - languageName: node - linkType: hard - "babel-preset-jest@npm:^29.6.3": version: 29.6.3 resolution: "babel-preset-jest@npm:29.6.3" @@ -22338,7 +22069,7 @@ __metadata: languageName: node linkType: hard -"expect@npm:^29.0.0, expect@npm:^29.5.0": +"expect@npm:^29.0.0": version: 29.5.0 resolution: "expect@npm:29.5.0" dependencies: @@ -26608,16 +26339,6 @@ __metadata: languageName: node linkType: hard -"jest-changed-files@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-changed-files@npm:29.5.0" - dependencies: - execa: ^5.0.0 - p-limit: ^3.1.0 - checksum: a67a7cb3c11f8f92bd1b7c79e84f724cbd11a9ad51f3cdadafe3ce7ee3c79ee50dbea128f920f5fddc807e9e4e83f5462143094391feedd959a77dd20ab96cf3 - languageName: node - linkType: hard - "jest-changed-files@npm:^29.7.0": version: 29.7.0 resolution: "jest-changed-files@npm:29.7.0" @@ -26656,34 +26377,6 @@ __metadata: languageName: node linkType: hard -"jest-circus@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-circus@npm:29.5.0" - dependencies: - "@jest/environment": ^29.5.0 - "@jest/expect": ^29.5.0 - "@jest/test-result": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - chalk: ^4.0.0 - co: ^4.6.0 - dedent: ^0.7.0 - is-generator-fn: ^2.0.0 - jest-each: ^29.5.0 - jest-matcher-utils: ^29.5.0 - jest-message-util: ^29.5.0 - jest-runtime: ^29.5.0 - jest-snapshot: ^29.5.0 - jest-util: ^29.5.0 - p-limit: ^3.1.0 - pretty-format: ^29.5.0 - pure-rand: ^6.0.0 - slash: ^3.0.0 - stack-utils: ^2.0.3 - checksum: 44ff5d06acedae6de6c866e20e3b61f83e29ab94cf9f960826e7e667de49c12dd9ab9dffd7fa3b7d1f9688a8b5bfb1ebebadbea69d9ed0d3f66af4a0ff8c2b27 - languageName: node - linkType: hard - "jest-circus@npm:^29.7.0": version: 29.7.0 resolution: "jest-circus@npm:29.7.0" @@ -26762,33 +26455,6 @@ __metadata: languageName: node linkType: hard -"jest-cli@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-cli@npm:29.5.0" - dependencies: - "@jest/core": ^29.5.0 - "@jest/test-result": ^29.5.0 - "@jest/types": ^29.5.0 - chalk: ^4.0.0 - exit: ^0.1.2 - graceful-fs: ^4.2.9 - import-local: ^3.0.2 - jest-config: ^29.5.0 - jest-util: ^29.5.0 - jest-validate: ^29.5.0 - prompts: ^2.0.1 - yargs: ^17.3.1 - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - bin: - jest: bin/jest.js - checksum: 39897bbbc0f0d8a6b975ab12fd13887eaa28d92e3dee9e0173a5cb913ae8cc2ae46e090d38c6d723e84d9d6724429cd08685b4e505fa447d31ca615630c7dbba - languageName: node - linkType: hard - "jest-cli@npm:^29.7.0": version: 29.7.0 resolution: "jest-cli@npm:29.7.0" @@ -26877,44 +26543,6 @@ __metadata: languageName: node linkType: hard -"jest-config@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-config@npm:29.5.0" - dependencies: - "@babel/core": ^7.11.6 - "@jest/test-sequencer": ^29.5.0 - "@jest/types": ^29.5.0 - babel-jest: ^29.5.0 - chalk: ^4.0.0 - ci-info: ^3.2.0 - deepmerge: ^4.2.2 - glob: ^7.1.3 - graceful-fs: ^4.2.9 - jest-circus: ^29.5.0 - jest-environment-node: ^29.5.0 - jest-get-type: ^29.4.3 - jest-regex-util: ^29.4.3 - jest-resolve: ^29.5.0 - jest-runner: ^29.5.0 - jest-util: ^29.5.0 - jest-validate: ^29.5.0 - micromatch: ^4.0.4 - parse-json: ^5.2.0 - pretty-format: ^29.5.0 - slash: ^3.0.0 - strip-json-comments: ^3.1.1 - peerDependencies: - "@types/node": "*" - ts-node: ">=9.0.0" - peerDependenciesMeta: - "@types/node": - optional: true - ts-node: - optional: true - checksum: c37c4dab964c54ab293d4e302d40b09687037ac9d00b88348ec42366970747feeaf265e12e3750cd3660b40c518d4031335eda11ac10b70b10e60797ebbd4b9c - languageName: node - linkType: hard - "jest-config@npm:^29.7.0": version: 29.7.0 resolution: "jest-config@npm:29.7.0" @@ -26977,7 +26605,7 @@ __metadata: languageName: node linkType: hard -"jest-diff@npm:^27.0.0, jest-diff@npm:^27.5.1": +"jest-diff@npm:^27.5.1": version: 27.5.1 resolution: "jest-diff@npm:27.5.1" dependencies: @@ -27031,15 +26659,6 @@ __metadata: languageName: node linkType: hard -"jest-docblock@npm:^29.4.3": - version: 29.4.3 - resolution: "jest-docblock@npm:29.4.3" - dependencies: - detect-newline: ^3.0.0 - checksum: e0e9df1485bb8926e5b33478cdf84b3387d9caf3658e7dc1eaa6dc34cb93dea0d2d74797f6e940f0233a88f3dadd60957f2288eb8f95506361f85b84bf8661df - languageName: node - linkType: hard - "jest-docblock@npm:^29.7.0": version: 29.7.0 resolution: "jest-docblock@npm:29.7.0" @@ -27075,19 +26694,6 @@ __metadata: languageName: node linkType: hard -"jest-each@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-each@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - chalk: ^4.0.0 - jest-get-type: ^29.4.3 - jest-util: ^29.5.0 - pretty-format: ^29.5.0 - checksum: b8b297534d25834c5d4e31e4c687359787b1e402519e42664eb704cc3a12a7a91a017565a75acb02e8cf9afd3f4eef3350bd785276bec0900184641b765ff7a5 - languageName: node - linkType: hard - "jest-each@npm:^29.7.0": version: 29.7.0 resolution: "jest-each@npm:29.7.0" @@ -27171,20 +26777,6 @@ __metadata: languageName: node linkType: hard -"jest-environment-node@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-environment-node@npm:29.5.0" - dependencies: - "@jest/environment": ^29.5.0 - "@jest/fake-timers": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - jest-mock: ^29.5.0 - jest-util: ^29.5.0 - checksum: 57981911cc20a4219b0da9e22b2e3c9f31b505e43f78e61c899e3227ded455ce1a3a9483842c69cfa4532f02cfb536ae0995bf245f9211608edacfc1e478d411 - languageName: node - linkType: hard - "jest-environment-node@npm:^29.7.0": version: 29.7.0 resolution: "jest-environment-node@npm:29.7.0" @@ -27296,29 +26888,6 @@ __metadata: languageName: node linkType: hard -"jest-haste-map@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-haste-map@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - "@types/graceful-fs": ^4.1.3 - "@types/node": "*" - anymatch: ^3.0.3 - fb-watchman: ^2.0.0 - fsevents: ^2.3.2 - graceful-fs: ^4.2.9 - jest-regex-util: ^29.4.3 - jest-util: ^29.5.0 - jest-worker: ^29.5.0 - micromatch: ^4.0.4 - walker: ^1.0.8 - dependenciesMeta: - fsevents: - optional: true - checksum: 3828ff7783f168e34be2c63887f82a01634261f605dcae062d83f979a61c37739e21b9607ecb962256aea3fbe5a530a1acee062d0026fcb47c607c12796cf3b7 - languageName: node - linkType: hard - "jest-haste-map@npm:^29.7.0": version: 29.7.0 resolution: "jest-haste-map@npm:29.7.0" @@ -27411,16 +26980,6 @@ __metadata: languageName: node linkType: hard -"jest-leak-detector@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-leak-detector@npm:29.5.0" - dependencies: - jest-get-type: ^29.4.3 - pretty-format: ^29.5.0 - checksum: 0fb845da7ac9cdfc9b3b2e35f6f623a41c547d7dc0103ceb0349013459d00de5870b5689a625e7e37f9644934b40e8f1dcdd5422d14d57470600350364676313 - languageName: node - linkType: hard - "jest-leak-detector@npm:^29.7.0": version: 29.7.0 resolution: "jest-leak-detector@npm:29.7.0" @@ -27565,17 +27124,6 @@ __metadata: languageName: node linkType: hard -"jest-mock@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-mock@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - "@types/node": "*" - jest-util: ^29.5.0 - checksum: 2a9cf07509948fa8608898c445f04fe4dd6e2049ff431e5531eee028c808d3ba3c67f226ac87b0cf383feaa1055776900d197c895e89783016886ac17a4ff10c - languageName: node - linkType: hard - "jest-mock@npm:^29.7.0": version: 29.7.0 resolution: "jest-mock@npm:29.7.0" @@ -27613,13 +27161,6 @@ __metadata: languageName: node linkType: hard -"jest-regex-util@npm:^29.4.3": - version: 29.4.3 - resolution: "jest-regex-util@npm:29.4.3" - checksum: 96fc7fc28cd4dd73a63c13a526202c4bd8b351d4e5b68b1a2a2c88da3308c2a16e26feaa593083eb0bac38cca1aa9dd05025412e7de013ba963fb8e66af22b8a - languageName: node - linkType: hard - "jest-regex-util@npm:^29.6.3": version: 29.6.3 resolution: "jest-regex-util@npm:29.6.3" @@ -27649,16 +27190,6 @@ __metadata: languageName: node linkType: hard -"jest-resolve-dependencies@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-resolve-dependencies@npm:29.5.0" - dependencies: - jest-regex-util: ^29.4.3 - jest-snapshot: ^29.5.0 - checksum: 479d2e5365d58fe23f2b87001e2e0adcbffe0147700e85abdec8f14b9703b0a55758c1929a9989e3f5d5e954fb88870ea4bfa04783523b664562fcf5f10b0edf - languageName: node - linkType: hard - "jest-resolve-dependencies@npm:^29.7.0": version: 29.7.0 resolution: "jest-resolve-dependencies@npm:29.7.0" @@ -27700,23 +27231,6 @@ __metadata: languageName: node linkType: hard -"jest-resolve@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-resolve@npm:29.5.0" - dependencies: - chalk: ^4.0.0 - graceful-fs: ^4.2.9 - jest-haste-map: ^29.5.0 - jest-pnp-resolver: ^1.2.2 - jest-util: ^29.5.0 - jest-validate: ^29.5.0 - resolve: ^1.20.0 - resolve.exports: ^2.0.0 - slash: ^3.0.0 - checksum: 9a125f3cf323ceef512089339d35f3ee37f79fe16a831fb6a26773ea6a229b9e490d108fec7af334142e91845b5996de8e7cdd85a4d8d617078737d804e29c8f - languageName: node - linkType: hard - "jest-resolve@npm:^29.7.0": version: 29.7.0 resolution: "jest-resolve@npm:29.7.0" @@ -27790,35 +27304,6 @@ __metadata: languageName: node linkType: hard -"jest-runner@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-runner@npm:29.5.0" - dependencies: - "@jest/console": ^29.5.0 - "@jest/environment": ^29.5.0 - "@jest/test-result": ^29.5.0 - "@jest/transform": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - chalk: ^4.0.0 - emittery: ^0.13.1 - graceful-fs: ^4.2.9 - jest-docblock: ^29.4.3 - jest-environment-node: ^29.5.0 - jest-haste-map: ^29.5.0 - jest-leak-detector: ^29.5.0 - jest-message-util: ^29.5.0 - jest-resolve: ^29.5.0 - jest-runtime: ^29.5.0 - jest-util: ^29.5.0 - jest-watcher: ^29.5.0 - jest-worker: ^29.5.0 - p-limit: ^3.1.0 - source-map-support: 0.5.13 - checksum: 437dea69c5dddca22032259787bac74790d5a171c9d804711415f31e5d1abfb64fa52f54a9015bb17a12b858fd0cf3f75ef6f3c9e94255a8596e179f707229c4 - languageName: node - linkType: hard - "jest-runner@npm:^29.7.0": version: 29.7.0 resolution: "jest-runner@npm:29.7.0" @@ -27911,36 +27396,6 @@ __metadata: languageName: node linkType: hard -"jest-runtime@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-runtime@npm:29.5.0" - dependencies: - "@jest/environment": ^29.5.0 - "@jest/fake-timers": ^29.5.0 - "@jest/globals": ^29.5.0 - "@jest/source-map": ^29.4.3 - "@jest/test-result": ^29.5.0 - "@jest/transform": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - chalk: ^4.0.0 - cjs-module-lexer: ^1.0.0 - collect-v8-coverage: ^1.0.0 - glob: ^7.1.3 - graceful-fs: ^4.2.9 - jest-haste-map: ^29.5.0 - jest-message-util: ^29.5.0 - jest-mock: ^29.5.0 - jest-regex-util: ^29.4.3 - jest-resolve: ^29.5.0 - jest-snapshot: ^29.5.0 - jest-util: ^29.5.0 - slash: ^3.0.0 - strip-bom: ^4.0.0 - checksum: 7af27bd9d54cf1c5735404cf8d76c6509d5610b1ec0106a21baa815c1aff15d774ce534ac2834bc440dccfe6348bae1885fd9a806f23a94ddafdc0f5bae4b09d - languageName: node - linkType: hard - "jest-runtime@npm:^29.7.0": version: 29.7.0 resolution: "jest-runtime@npm:29.7.0" @@ -28060,37 +27515,6 @@ __metadata: languageName: node linkType: hard -"jest-snapshot@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-snapshot@npm:29.5.0" - dependencies: - "@babel/core": ^7.11.6 - "@babel/generator": ^7.7.2 - "@babel/plugin-syntax-jsx": ^7.7.2 - "@babel/plugin-syntax-typescript": ^7.7.2 - "@babel/traverse": ^7.7.2 - "@babel/types": ^7.3.3 - "@jest/expect-utils": ^29.5.0 - "@jest/transform": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/babel__traverse": ^7.0.6 - "@types/prettier": ^2.1.5 - babel-preset-current-node-syntax: ^1.0.0 - chalk: ^4.0.0 - expect: ^29.5.0 - graceful-fs: ^4.2.9 - jest-diff: ^29.5.0 - jest-get-type: ^29.4.3 - jest-matcher-utils: ^29.5.0 - jest-message-util: ^29.5.0 - jest-util: ^29.5.0 - natural-compare: ^1.4.0 - pretty-format: ^29.5.0 - semver: ^7.3.5 - checksum: fe5df54122ed10eed625de6416a45bc4958d5062b018f05b152bf9785ab7f355dcd55e40cf5da63895bf8278f8d7b2bb4059b2cfbfdee18f509d455d37d8aa2b - languageName: node - linkType: hard - "jest-snapshot@npm:^29.7.0": version: 29.7.0 resolution: "jest-snapshot@npm:29.7.0" @@ -28216,20 +27640,6 @@ __metadata: languageName: node linkType: hard -"jest-validate@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-validate@npm:29.5.0" - dependencies: - "@jest/types": ^29.5.0 - camelcase: ^6.2.0 - chalk: ^4.0.0 - jest-get-type: ^29.4.3 - leven: ^3.1.0 - pretty-format: ^29.5.0 - checksum: 43ca5df7cb75572a254ac3e92fbbe7be6b6a1be898cc1e887a45d55ea003f7a112717d814a674d37f9f18f52d8de40873c8f084f17664ae562736c78dd44c6a1 - languageName: node - linkType: hard - "jest-validate@npm:^29.7.0": version: 29.7.0 resolution: "jest-validate@npm:29.7.0" @@ -28289,22 +27699,6 @@ __metadata: languageName: node linkType: hard -"jest-watcher@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-watcher@npm:29.5.0" - dependencies: - "@jest/test-result": ^29.5.0 - "@jest/types": ^29.5.0 - "@types/node": "*" - ansi-escapes: ^4.2.1 - chalk: ^4.0.0 - emittery: ^0.13.1 - jest-util: ^29.5.0 - string-length: ^4.0.1 - checksum: 62303ac7bdc7e61a8b4239a239d018f7527739da2b2be6a81a7be25b74ca769f1c43ee8558ce8e72bb857245c46d6e03af331227ffb00a57280abb2a928aa776 - languageName: node - linkType: hard - "jest-watcher@npm:^29.7.0": version: 29.7.0 resolution: "jest-watcher@npm:29.7.0" @@ -28353,18 +27747,6 @@ __metadata: languageName: node linkType: hard -"jest-worker@npm:^29.5.0": - version: 29.5.0 - resolution: "jest-worker@npm:29.5.0" - dependencies: - "@types/node": "*" - jest-util: ^29.5.0 - merge-stream: ^2.0.0 - supports-color: ^8.0.0 - checksum: 1151a1ae3602b1ea7c42a8f1efe2b5a7bf927039deaa0827bf978880169899b705744e288f80a63603fb3fc2985e0071234986af7dc2c21c7a64333d8777c7c9 - languageName: node - linkType: hard - "jest-worker@npm:^29.7.0": version: 29.7.0 resolution: "jest-worker@npm:29.7.0" @@ -28407,26 +27789,7 @@ __metadata: languageName: node linkType: hard -"jest@npm:^29.5.0": - version: 29.5.0 - resolution: "jest@npm:29.5.0" - dependencies: - "@jest/core": ^29.5.0 - "@jest/types": ^29.5.0 - import-local: ^3.0.2 - jest-cli: ^29.5.0 - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - bin: - jest: bin/jest.js - checksum: a8ff2eb0f421623412236e23cbe67c638127fffde466cba9606bc0c0553b4c1e5cb116d7e0ef990b5d1712851652c8ee461373b578df50857fe635b94ff455d5 - languageName: node - linkType: hard - -"jest@npm:^29.7.0": +"jest@npm:^29.5.0, jest@npm:^29.7.0": version: 29.7.0 resolution: "jest@npm:29.7.0" dependencies: @@ -35238,7 +34601,7 @@ __metadata: languageName: node linkType: hard -"pretty-format@npm:^27.0.0, pretty-format@npm:^27.4.6, pretty-format@npm:^27.5.1": +"pretty-format@npm:^27.4.6, pretty-format@npm:^27.5.1": version: 27.5.1 resolution: "pretty-format@npm:27.5.1" dependencies: From dc23335480d0202f5a3f318458c94031fc2eef2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 28 Oct 2024 12:52:27 +0100 Subject: [PATCH 03/15] fix: Retry JSON queries on Vertex AI --- packages/navie/package.json | 1 + .../google-vertexai-completion-service.ts | 40 ++++++++++++++----- ...google-vertexai-completion-service.spec.ts | 35 +++++++++++++--- yarn.lock | 1 + 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/packages/navie/package.json b/packages/navie/package.json index 3c7c98ed3b..2a54dca0ba 100644 --- a/packages/navie/package.json +++ b/packages/navie/package.json @@ -49,6 +49,7 @@ "langchain": "^0.2.16", "mermaid": "^9", "openai": "^4.56.0", + "p-retry": "4", "zod": "^3.23.8" } } diff --git a/packages/navie/src/services/google-vertexai-completion-service.ts b/packages/navie/src/services/google-vertexai-completion-service.ts index 9d66ae5cd3..c283c3731b 100644 --- a/packages/navie/src/services/google-vertexai-completion-service.ts +++ b/packages/navie/src/services/google-vertexai-completion-service.ts @@ -1,8 +1,10 @@ +import assert from 'node:assert'; import { warn } from 'node:console'; import { isNativeError } from 'node:util/types'; import { ChatVertexAI, type ChatVertexAIInput } from '@langchain/google-vertexai-web'; import { zodResponseFormat } from 'openai/helpers/zod'; +import pRetry from 'p-retry'; import { z } from 'zod'; import Trajectory from '../lib/trajectory'; @@ -64,19 +66,37 @@ export default class GoogleVertexAICompletionService implements CompletionServic }, ]); - for (const message of sentMessages) this.trajectory.logSentMessage(message); + const processResponse = async () => { + for (const message of sentMessages) this.trajectory.logSentMessage(message); - const response = await model.invoke(sentMessages.map(convertToMessage)); + const response = await model.invoke(sentMessages.map(convertToMessage)); - this.trajectory.logReceivedMessage({ - role: 'assistant', - content: JSON.stringify(response), - }); + this.trajectory.logReceivedMessage({ + role: 'assistant', + content: JSON.stringify(response), + }); - const sanitizedContent = response.content.toString().replace(/^`{3,}[^\s]*?$/gm, ''); - const parsed = JSON.parse(sanitizedContent) as unknown; - schema.parse(parsed); - return parsed; + const sanitizedContent = response.content.toString().replace(/^`{3,}[^\s]*?$/gm, ''); + try { + const parsed = JSON.parse(sanitizedContent) as unknown; + schema.parse(parsed); + return parsed; + } catch (e) { + assert(isNativeError(e)); + (e as Error & { response: unknown })['response'] = response; + throw e; + } + }; + + return await pRetry(processResponse, { + retries: CompletionRetries, + minTimeout: CompletionRetryDelay, + randomize: true, + onFailedAttempt: (err) => { + warn(`Failed to complete after ${err.attemptNumber} attempt(s): ${String(err)}`); + if ('response' in err) warn(`Response: ${JSON.stringify(err.response)}`); + }, + }); } async *complete(messages: readonly Message[], options?: { temperature?: number }): Completion { diff --git a/packages/navie/test/services/google-vertexai-completion-service.spec.ts b/packages/navie/test/services/google-vertexai-completion-service.spec.ts index 0d45fc9d21..0f5501a808 100644 --- a/packages/navie/test/services/google-vertexai-completion-service.spec.ts +++ b/packages/navie/test/services/google-vertexai-completion-service.spec.ts @@ -50,18 +50,17 @@ describe('GoogleVertexAICompletionService', () => { }); it('retries on errors', async () => { - jest.useFakeTimers(); mockCompletion('hello world'); responseMock.mockImplementationOnce(() => { throw new Error('error'); }); const messages: Message[] = []; - // eslint-disable-next-line jest/valid-expect - const result = expect(collect(service.complete(messages))).resolves.toEqual(['hello world']); - await jest.advanceTimersByTimeAsync(1000); + jest.useFakeTimers(); + const result = collect(service.complete(messages)); + await jest.runAllTimersAsync(); jest.useRealTimers(); - return result; + expect(await result).toEqual(['hello world']); }); }); @@ -73,6 +72,32 @@ describe('GoogleVertexAICompletionService', () => { const result = await service.json(messages, schema); expect(result).toEqual({ foo: 'bar' }); }); + + it('retries on errors', async () => { + jest.useFakeTimers(); + mockCompletion('{"foo": "bar"}'); + responseMock.mockImplementationOnce(() => { + throw new Error('error'); + }); + const schema = z.object({ foo: z.string() }); + const messages: Message[] = []; + const result = service.json(messages, schema); + await jest.runAllTimersAsync(); + expect(await result).toEqual({ foo: 'bar' }); + jest.useRealTimers(); + }); + + it('retries on bad JSON', async () => { + jest.useFakeTimers(); + mockCompletion('{"foo": "bar"}'); + responseMock.mockReturnValueOnce('{"foo": "bar"'); + const schema = z.object({ foo: z.string() }); + const messages: Message[] = []; + const result = service.json(messages, schema); + await jest.runAllTimersAsync(); + expect(await result).toEqual({ foo: 'bar' }); + jest.useRealTimers(); + }); }); describe('miniModelName', () => { diff --git a/yarn.lock b/yarn.lock index 531892071d..71768c91ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -479,6 +479,7 @@ __metadata: langchain: ^0.2.16 mermaid: ^9 openai: ^4.56.0 + p-retry: 4 ts-jest: ^29.2.5 ts-node: ^10.4.0 tsc: ^2.0.3 From b66c64b19c53d308035e86629d1061b4d8054081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 28 Oct 2024 12:58:19 +0100 Subject: [PATCH 04/15] fix: Instruct the LLM to be briefer in the vector terms I noticed sometimes the LLM tries to put a lot in the `instructions` part of the vector terms response; this part is only meant to direct reasoning and we're ultimately throwing it out, so it doesn't make sense to be very verbose. It also appears to hit against some limits, at least on Gemini. --- packages/navie/src/services/vector-terms-service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/navie/src/services/vector-terms-service.ts b/packages/navie/src/services/vector-terms-service.ts index 4704a3fedb..38b5752e85 100644 --- a/packages/navie/src/services/vector-terms-service.ts +++ b/packages/navie/src/services/vector-terms-service.ts @@ -17,13 +17,14 @@ The developer asks a question using natural language. This question must be conv 1) Separate the user's question into "context" and "instructions". The "context" defines the background information that will match code in the code base. The "instructions" are the statement about what the user wants you to do or what they want to find out. + Be brief in the "instructions" part. 2) From here on, only consider the "context" part of the question. 3) Expand the list of "context" words to include synonyms and related terms. 4) Optionally choose a small number of search terms which are MOST SELECTIVE. The MOST SELECTIVE match terms should be words that will match a feature or domain model object in the code base. They should be the most distinctive words in the question. You will prefix the MOST SELECTIVE terms with a '+'. 5) In order for keywords to match optimally, compound words MUST additionally be segmented into individual search terms. This is additionally very important for compound words containing acronyms. - E.g., "httpserver" would become "http", "server", "httpserver". "xmlserializer" -> "xml", "serializer", "xmlserializer". "jsonserializer" -> "json", "serializer", "jsonserializer". etc. + E.g., "httpserver" would become "http", "server", "httpserver". "xmlserializer" -> "xml", "serializer", "xmlserializer". "jsonserializer" -> "json", "serializer", "jsonserializer". etc. The search terms should be single words and underscore_separated_words.`; From 39240beb8706fb25acc0978a9f309c72ac17e771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 28 Oct 2024 13:56:11 +0100 Subject: [PATCH 05/15] Raise temperature in Vertex AI when retrying This might help avoid recitation errors on retry. --- .../src/services/google-vertexai-completion-service.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/navie/src/services/google-vertexai-completion-service.ts b/packages/navie/src/services/google-vertexai-completion-service.ts index c283c3731b..bc7fd41a24 100644 --- a/packages/navie/src/services/google-vertexai-completion-service.ts +++ b/packages/navie/src/services/google-vertexai-completion-service.ts @@ -66,10 +66,12 @@ export default class GoogleVertexAICompletionService implements CompletionServic }, ]); + let { temperature } = model; + const processResponse = async () => { for (const message of sentMessages) this.trajectory.logSentMessage(message); - const response = await model.invoke(sentMessages.map(convertToMessage)); + const response = await model.invoke(sentMessages.map(convertToMessage), { temperature }); this.trajectory.logReceivedMessage({ role: 'assistant', @@ -95,6 +97,7 @@ export default class GoogleVertexAICompletionService implements CompletionServic onFailedAttempt: (err) => { warn(`Failed to complete after ${err.attemptNumber} attempt(s): ${String(err)}`); if ('response' in err) warn(`Response: ${JSON.stringify(err.response)}`); + temperature += 0.1; }, }); } @@ -102,6 +105,7 @@ export default class GoogleVertexAICompletionService implements CompletionServic async *complete(messages: readonly Message[], options?: { temperature?: number }): Completion { const usage = new Usage(); const model = this.buildModel(options); + let { temperature } = model; const sentMessages: Message[] = mergeSystemMessages(messages); const tokens = new Array(); for (const message of sentMessages) this.trajectory.logSentMessage(message); @@ -110,7 +114,7 @@ export default class GoogleVertexAICompletionService implements CompletionServic for (let attempt = 0; attempt < maxAttempts; attempt += 1) { try { // eslint-disable-next-line no-await-in-loop - const response = await model.stream(sentMessages.map(convertToMessage)); + const response = await model.stream(sentMessages.map(convertToMessage), { temperature }); // eslint-disable-next-line @typescript-eslint/naming-convention, no-await-in-loop for await (const { content, usage_metadata } of response) { @@ -129,6 +133,7 @@ export default class GoogleVertexAICompletionService implements CompletionServic break; } catch (cause) { + temperature += 0.1; if (attempt < maxAttempts - 1 && tokens.length === 0) { const nextAttempt = CompletionRetryDelay * 2 ** attempt; warn(`Received ${JSON.stringify(cause)}, retrying in ${nextAttempt}ms`); From 14d819153855f3cfaa34fb36fd9bbc9eb419776d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Mon, 28 Oct 2024 15:08:55 +0100 Subject: [PATCH 06/15] chore: Force whatwg-url dependency to a recent version The old version dropped deprecation warnings for punycode module. --- package.json | 3 +- yarn.lock | 122 +++++++-------------------------------------------- 2 files changed, 17 insertions(+), 108 deletions(-) diff --git a/package.json b/package.json index e4c4e06e57..8a9ff564ed 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "puppeteer": "^19.7.2" }, "resolutions": { - "web-auth-library": "getappmap/web-auth-library#v1.0.3-cjs" + "web-auth-library": "getappmap/web-auth-library#v1.0.3-cjs", + "whatwg-url": "14.0.0" } } diff --git a/yarn.lock b/yarn.lock index 71768c91ef..829a648969 100644 --- a/yarn.lock +++ b/yarn.lock @@ -29635,13 +29635,6 @@ __metadata: languageName: node linkType: hard -"lodash.sortby@npm:^4.7.0": - version: 4.7.0 - resolution: "lodash.sortby@npm:4.7.0" - checksum: db170c9396d29d11fe9a9f25668c4993e0c1331bcb941ddbd48fb76f492e732add7f2a47cfdf8e9d740fa59ac41bbfaf931d268bc72aab3ab49e9f89354d718c - languageName: node - linkType: hard - "lodash.template@npm:^4.5.0": version: 4.5.0 resolution: "lodash.template@npm:4.5.0" @@ -29689,7 +29682,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.11, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.17.3, lodash@npm:^4.17.4, lodash@npm:^4.7.0": +"lodash@npm:^4.17.11, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:^4.17.3, lodash@npm:^4.17.4": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 @@ -35062,10 +35055,10 @@ __metadata: languageName: node linkType: hard -"punycode@npm:^2.1.0, punycode@npm:^2.1.1": - version: 2.1.1 - resolution: "punycode@npm:2.1.1" - checksum: 823bf443c6dd14f669984dea25757b37993f67e8d94698996064035edd43bed8a5a17a9f12e439c2b35df1078c6bec05a6c86e336209eb1061e8025c481168e8 +"punycode@npm:^2.1.0, punycode@npm:^2.1.1, punycode@npm:^2.3.1": + version: 2.3.1 + resolution: "punycode@npm:2.3.1" + checksum: bb0a0ceedca4c3c57a9b981b90601579058903c62be23c5e8e843d2c2d4148a3ecf029d5133486fb0e1822b098ba8bba09e89d6b21742d02fa26bda6441a6fb2 languageName: node linkType: hard @@ -39418,37 +39411,12 @@ resolve@1.1.7: languageName: node linkType: hard -"tr46@npm:^1.0.1": - version: 1.0.1 - resolution: "tr46@npm:1.0.1" - dependencies: - punycode: ^2.1.0 - checksum: 96d4ed46bc161db75dbf9247a236ea0bfcaf5758baae6749e92afab0bc5a09cb59af21788ede7e55080f2bf02dce3e4a8f2a484cc45164e29f4b5e68f7cbcc1a - languageName: node - linkType: hard - -"tr46@npm:^2.1.0": - version: 2.1.0 - resolution: "tr46@npm:2.1.0" - dependencies: - punycode: ^2.1.1 - checksum: ffe6049b9dca3ae329b059aada7f515b0f0064c611b39b51ff6b53897e954650f6f63d9319c6c008d36ead477c7b55e5f64c9dc60588ddc91ff720d64eb710b3 - languageName: node - linkType: hard - -"tr46@npm:^3.0.0": - version: 3.0.0 - resolution: "tr46@npm:3.0.0" +"tr46@npm:^5.0.0": + version: 5.0.0 + resolution: "tr46@npm:5.0.0" dependencies: - punycode: ^2.1.1 - checksum: 44c3cc6767fb800490e6e9fd64fd49041aa4e49e1f6a012b34a75de739cc9ed3a6405296072c1df8b6389ae139c5e7c6496f659cfe13a04a4bff3a1422981270 - languageName: node - linkType: hard - -"tr46@npm:~0.0.3": - version: 0.0.3 - resolution: "tr46@npm:0.0.3" - checksum: 726321c5eaf41b5002e17ffbd1fb7245999a073e8979085dacd47c4b4e8068ff5777142fc6726d6ca1fd2ff16921b48788b87225cbc57c72636f6efa8efbffe3 + punycode: ^2.3.1 + checksum: 8d8b021f8e17675ebf9e672c224b6b6cfdb0d5b92141349e9665c14a2501c54a298d11264bbb0b17b447581e1e83d4fc3c038c929f3d210e3964d4be47460288 languageName: node linkType: hard @@ -41292,13 +41260,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"webidl-conversions@npm:^3.0.0": - version: 3.0.1 - resolution: "webidl-conversions@npm:3.0.1" - checksum: c92a0a6ab95314bde9c32e1d0a6dfac83b578f8fa5f21e675bc2706ed6981bc26b7eb7e6a1fab158e5ce4adf9caa4a0aee49a52505d4d13c7be545f15021b17c - languageName: node - linkType: hard - "webidl-conversions@npm:^4.0.2": version: 4.0.2 resolution: "webidl-conversions@npm:4.0.2" @@ -41679,66 +41640,13 @@ typescript@~4.4.3: languageName: node linkType: hard -"whatwg-url@npm:^11.0.0": - version: 11.0.0 - resolution: "whatwg-url@npm:11.0.0" +"whatwg-url@npm:14.0.0": + version: 14.0.0 + resolution: "whatwg-url@npm:14.0.0" dependencies: - tr46: ^3.0.0 + tr46: ^5.0.0 webidl-conversions: ^7.0.0 - checksum: ed4826aaa57e66bb3488a4b25c9cd476c46ba96052747388b5801f137dd740b73fde91ad207d96baf9f17fbcc80fc1a477ad65181b5eb5fa718d27c69501d7af - languageName: node - linkType: hard - -"whatwg-url@npm:^5.0.0": - version: 5.0.0 - resolution: "whatwg-url@npm:5.0.0" - dependencies: - tr46: ~0.0.3 - webidl-conversions: ^3.0.0 - checksum: b8daed4ad3356cc4899048a15b2c143a9aed0dfae1f611ebd55073310c7b910f522ad75d727346ad64203d7e6c79ef25eafd465f4d12775ca44b90fa82ed9e2c - languageName: node - linkType: hard - -"whatwg-url@npm:^6.4.1": - version: 6.5.0 - resolution: "whatwg-url@npm:6.5.0" - dependencies: - lodash.sortby: ^4.7.0 - tr46: ^1.0.1 - webidl-conversions: ^4.0.2 - checksum: a10bd5e29f4382cd19789c2a7bbce25416e606b6fefc241c7fe34a2449de5bc5709c165bd13634eda433942d917ca7386a52841780b82dc37afa8141c31a8ebd - languageName: node - linkType: hard - -"whatwg-url@npm:^7.0.0": - version: 7.1.0 - resolution: "whatwg-url@npm:7.1.0" - dependencies: - lodash.sortby: ^4.7.0 - tr46: ^1.0.1 - webidl-conversions: ^4.0.2 - checksum: fecb07c87290b47d2ec2fb6d6ca26daad3c9e211e0e531dd7566e7ff95b5b3525a57d4f32640ad4adf057717e0c215731db842ad761e61d947e81010e05cf5fd - languageName: node - linkType: hard - -"whatwg-url@npm:^8.0.0, whatwg-url@npm:^8.5.0": - version: 8.7.0 - resolution: "whatwg-url@npm:8.7.0" - dependencies: - lodash: ^4.7.0 - tr46: ^2.1.0 - webidl-conversions: ^6.1.0 - checksum: a87abcc6cefcece5311eb642858c8fdb234e51ec74196bfacf8def2edae1bfbffdf6acb251646ed6301f8cee44262642d8769c707256125a91387e33f405dd1e - languageName: node - linkType: hard - -"whatwg-url@npm:^9.0.0": - version: 9.1.0 - resolution: "whatwg-url@npm:9.1.0" - dependencies: - tr46: ^2.1.0 - webidl-conversions: ^6.1.0 - checksum: cfee81bb7f87036e1151da15cefd3076fa97a4a4a658c4b58f6e74891acf25f180aa955e761cda77995f6e260b8dc3c4326ebc83d539ed978a50062c6b3bd0d1 + checksum: 4b5887e50f786583bead70916413e67a381d2126899b9eb5c67ce664bba1e7ec07cdff791404581ce73c6190d83c359c9ca1d50711631217905db3877dec075c languageName: node linkType: hard From 62ab2faf2079be895196cae2b1d50132dd29e546 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 28 Oct 2024 17:24:42 -0400 Subject: [PATCH 07/15] fix: Adjust the first line of a patch when it has 0 indent Fixes https://github.com/getappmap/appmap-js/issues/2095 --- packages/cli/src/rpc/file/applyFileUpdate.ts | 104 +++++++++++++++--- .../unit/rpc/file/applyFileUpdate.spec.ts | 53 +++++---- .../apply.yml | 22 ++++ .../expected.txt | 12 ++ .../original.txt | 11 ++ .../missing-firstline-indent/apply.yml | 22 ++++ .../missing-firstline-indent/expected.txt | 12 ++ .../missing-firstline-indent/original.txt | 11 ++ 8 files changed, 208 insertions(+), 39 deletions(-) create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/apply.yml create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/expected.txt create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/original.txt create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/apply.yml create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/expected.txt create mode 100644 packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/original.txt diff --git a/packages/cli/src/rpc/file/applyFileUpdate.ts b/packages/cli/src/rpc/file/applyFileUpdate.ts index 4be63954bf..78df8db52a 100644 --- a/packages/cli/src/rpc/file/applyFileUpdate.ts +++ b/packages/cli/src/rpc/file/applyFileUpdate.ts @@ -1,7 +1,8 @@ -import { verbose } from '../../utils'; import { readFile, writeFile } from 'fs/promises'; -import { warn } from 'console'; import assert from 'assert'; +import makeDebug from 'debug'; + +const debug = makeDebug('appmap:cli:file-update'); function findLineMatch( haystack: readonly string[], @@ -47,35 +48,102 @@ function makeWhitespaceAdjuster(to: string, from: string) { return adjuster; } +type MatchResult = { + index: number; + length: number; + whitespaceAdjuster: (s: string) => string; + whitespaceAdjusterDescription: string; +}; + +function matchFileUpdate(fileLines: string[], originalLines: string[]): MatchResult | undefined { + const match = findLineMatch(fileLines, originalLines); + if (!match) return undefined; + + const [index, length] = match; + const nonEmptyIndex = originalLines.findIndex((s) => s.trim()); + const adjustWhitespace = makeWhitespaceAdjuster( + fileLines[index + nonEmptyIndex], + originalLines[nonEmptyIndex] + ); + + return { + index, + length, + whitespaceAdjuster: adjustWhitespace, + whitespaceAdjusterDescription: adjustWhitespace.desc, + }; +} + +function searchForFileUpdate( + whitespaceAdjustments: string[], + fileLines: string[], + originalLines: string[] +): [MatchResult, string] | undefined { + for (const whitespaceAdjustment of whitespaceAdjustments) { + const adjustedOriginalLines = [...originalLines]; + adjustedOriginalLines[0] = whitespaceAdjustment + adjustedOriginalLines[0]; + const match = matchFileUpdate(fileLines, adjustedOriginalLines); + if (match) return [match, whitespaceAdjustment]; + } + + return undefined; +} + export default async function applyFileUpdate( file: string, original: string, modified: string ): Promise { - // Read the original file const fileContents = await readFile(file, 'utf-8'); const fileLines = fileContents.split('\n'); - const originalLines = original.split('\n'); - const match = findLineMatch(fileLines, originalLines); - if (!match) return [`[file-update] Failed to find match for ${file}.\n`]; + if (fileLines.length === 0) { + debug(`File is empty. Skipping.`); + return undefined; + } + if (originalLines.length === 0) { + debug(`Original text is empty. Skipping.`); + return undefined; + } - const [index, length] = match; + const firstLineLeadingWhitespace = originalLines[0].match(/^\s*/)?.[0]; + let whitespaceAdjustments: Set; - const nonEmptyIndex = originalLines.findIndex((s) => s.trim()); - const adjustWhitespace = makeWhitespaceAdjuster( - fileLines[index + nonEmptyIndex], - originalLines[nonEmptyIndex] + if (!firstLineLeadingWhitespace) { + debug( + `No leading whitespace found in the first line of the original text. Will attempt a fuzzy match.` + ); + const fileLinesMatchingFirstOriginalLine = new Set( + fileLines.filter((line) => line.includes(originalLines[0])) + ); + whitespaceAdjustments = new Set( + Array.from(fileLinesMatchingFirstOriginalLine).map((line) => line.match(/^\s*/)?.[0] ?? '') + ); + } else { + whitespaceAdjustments = new Set(['']); + } + + const searchResult = searchForFileUpdate( + Array.from(whitespaceAdjustments), + fileLines, + originalLines ); + if (!searchResult) { + debug(`No match found for the original text.`); + return undefined; + } - if (verbose()) - warn( - `[file-update] Found match at line ${index + 1}, whitespace adjustment: ${ - adjustWhitespace.desc - }\n` - ); - fileLines.splice(index, length, ...modified.split('\n').map(adjustWhitespace)); + const [match, leadingWhitespace] = searchResult; + const adjustedModified = [...modified.split('\n')]; + adjustedModified[0] = leadingWhitespace + adjustedModified[0]; + + debug( + `[file-update] Found match at line ${match.index + 1}, whitespace adjustment: ${ + match.whitespaceAdjusterDescription + }\n` + ); + fileLines.splice(match.index, match.length, ...adjustedModified.map(match.whitespaceAdjuster)); await writeFile(file, fileLines.join('\n'), 'utf8'); } diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate.spec.ts b/packages/cli/tests/unit/rpc/file/applyFileUpdate.spec.ts index 79dd74f21c..3b07c9b834 100644 --- a/packages/cli/tests/unit/rpc/file/applyFileUpdate.spec.ts +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate.spec.ts @@ -5,7 +5,6 @@ import tmp from 'tmp'; import applyFileUpdate from '../../../../src/rpc/file/applyFileUpdate'; import { load } from 'js-yaml'; -import assert from 'node:assert'; type Change = { file: string; @@ -25,30 +24,34 @@ describe(applyFileUpdate, () => { process.chdir(startCwd); }); - const example = (name: string) => async () => { - expect.assertions(1); + const example = + (name: string, expectedAssertions = 1) => + async (): Promise => { + expect.assertions(expectedAssertions); - const fixtureDir = join(__dirname, 'applyFileUpdate', name); - await cp(fixtureDir, process.cwd(), { recursive: true }); - const applyStr = await readFile('apply.yml', 'utf8'); - const change = load(applyStr) as Change; - const { file, original, modified } = change; + const fixtureDir = join(__dirname, 'applyFileUpdate', name); + await cp(fixtureDir, process.cwd(), { recursive: true }); + const applyStr = await readFile('apply.yml', 'utf8'); + const change = load(applyStr) as Change; + const { file, original, modified } = change; - await applyFileUpdate(file, original, modified); + await applyFileUpdate(file, original, modified); - const updatedStr = await readFile('original.txt', 'utf8'); - const updated = updatedStr - .split('\n') - .map((line) => line.trim()) - .join('\n'); - const expectedStr = await readFile('expected.txt', 'utf8'); - const expected = expectedStr - .split('\n') - .map((line) => line.trim()) - .join('\n'); + const updatedStr = await readFile('original.txt', 'utf8'); + const updated = updatedStr + .split('\n') + .map((line) => line.trim()) + .join('\n'); + const expectedStr = await readFile('expected.txt', 'utf8'); + const expected = expectedStr + .split('\n') + .map((line) => line.trim()) + .join('\n'); - expect(updated).toEqual(expected); - }; + expect(updated).toEqual(expected); + + return updatedStr; + }; it('correctly applies an update even with broken whitespace', example('whitespace-mismatch')); it('correctly applies an update even with trailing newlines', example('trailing-newlines')); @@ -57,4 +60,12 @@ describe(applyFileUpdate, () => { 'correctly applies an update even when there are repeated similar but mismatching lines', example('mismatched-similar') ); + it('compensates for missing indentation in the first line', example('missing-firstline-indent')); + it('compensates for missing indentation in the first line', async () => { + const updated = await example('missing-firstline-indent-must-match', 4)(); + const updatedLines = updated.split('\n'); + expect(updatedLines[0]).toEqual(' # hello'); + expect(updatedLines[1]).toEqual(' def _bind_to_schema(self, field_name, schema):'); + expect(updatedLines[2]).toEqual(' super()._bind_to_schema(field_name, schema)'); + }); }); diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/apply.yml b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/apply.yml new file mode 100644 index 0000000000..bce6255b16 --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/apply.yml @@ -0,0 +1,22 @@ +file: original.txt +original: |- + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields +modified: |- + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + if hasattr(field, '_bind_to_schema'): + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/expected.txt b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/expected.txt new file mode 100644 index 0000000000..fa8efc4e5c --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/expected.txt @@ -0,0 +1,12 @@ + # hello + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + if hasattr(field, '_bind_to_schema'): + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields + diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/original.txt b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/original.txt new file mode 100644 index 0000000000..2dc7a4f1e2 --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent-must-match/original.txt @@ -0,0 +1,11 @@ + # hello + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields + diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/apply.yml b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/apply.yml new file mode 100644 index 0000000000..cb287e8987 --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/apply.yml @@ -0,0 +1,22 @@ +file: original.txt +original: |- + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields +modified: |- + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + if hasattr(field, '_bind_to_schema'): + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/expected.txt b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/expected.txt new file mode 100644 index 0000000000..fa8efc4e5c --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/expected.txt @@ -0,0 +1,12 @@ + # hello + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + if hasattr(field, '_bind_to_schema'): + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields + diff --git a/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/original.txt b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/original.txt new file mode 100644 index 0000000000..2dc7a4f1e2 --- /dev/null +++ b/packages/cli/tests/unit/rpc/file/applyFileUpdate/missing-firstline-indent/original.txt @@ -0,0 +1,11 @@ + # hello + def _bind_to_schema(self, field_name, schema): + super()._bind_to_schema(field_name, schema) + new_tuple_fields = [] + for field in self.tuple_fields: + field = copy.deepcopy(field) + field._bind_to_schema(field_name, self) + new_tuple_fields.append(field) + + self.tuple_fields = new_tuple_fields + From 0222ddb219fe22ad854b8f20eccb52a27fbf11f0 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 21 Oct 2024 18:05:20 -0400 Subject: [PATCH 08/15] feat: @appland/search package --- appmap.yml | 4 + packages/search/.eslintrc.js | 47 ++ packages/search/.gitignore | 2 + packages/search/jest.config.js | 11 + packages/search/package.json | 57 ++ packages/search/src/build-file-index.ts | 71 ++ packages/search/src/build-snippet-index.ts | 57 ++ packages/search/src/cli.ts | 126 ++++ packages/search/src/file-index.ts | 103 +++ packages/search/src/file-type.ts | 114 +++ packages/search/src/git.ts | 172 +++++ packages/search/src/index.ts | 9 + packages/search/src/ioutil.ts | 15 + packages/search/src/project-files.ts | 110 +++ packages/search/src/query-keywords.ts | 76 ++ packages/search/src/snippet-index.ts | 118 +++ packages/search/src/split-camelized.ts | 71 ++ packages/search/src/splitter.ts | 76 ++ packages/search/src/tokenize.ts | 104 +++ packages/search/test/file-index.spec.ts | 49 ++ packages/search/test/fixtures/source/sample.c | 30 + .../search/test/fixtures/source/sample.cs | 75 ++ .../test/fixtures/source/sample.generic | 103 +++ .../search/test/fixtures/source/sample.go | 36 + .../search/test/fixtures/source/sample.java | 21 + .../search/test/fixtures/source/sample.kt | 23 + .../search/test/fixtures/source/sample.php | 18 + .../search/test/fixtures/source/sample.py | 9 + .../search/test/fixtures/source/sample.rb | 16 + .../search/test/fixtures/source/sample.rs | 33 + .../search/test/fixtures/source/sample.ts | 46 ++ packages/search/test/project-files.spec.ts | 102 +++ packages/search/test/query-keywords.spec.ts | 83 +++ packages/search/test/snippet-index.spec.ts | 84 +++ packages/search/test/split-camelized.ts | 31 + packages/search/test/symbols.spec.ts | 195 +++++ packages/search/test/tokenize.spec.ts | 37 + packages/search/tsconfig.json | 11 + yarn.lock | 702 +++++++++++++++++- 39 files changed, 3040 insertions(+), 7 deletions(-) create mode 100644 packages/search/.eslintrc.js create mode 100644 packages/search/.gitignore create mode 100644 packages/search/jest.config.js create mode 100644 packages/search/package.json create mode 100644 packages/search/src/build-file-index.ts create mode 100644 packages/search/src/build-snippet-index.ts create mode 100644 packages/search/src/cli.ts create mode 100644 packages/search/src/file-index.ts create mode 100644 packages/search/src/file-type.ts create mode 100644 packages/search/src/git.ts create mode 100644 packages/search/src/index.ts create mode 100644 packages/search/src/ioutil.ts create mode 100644 packages/search/src/project-files.ts create mode 100644 packages/search/src/query-keywords.ts create mode 100644 packages/search/src/snippet-index.ts create mode 100644 packages/search/src/split-camelized.ts create mode 100644 packages/search/src/splitter.ts create mode 100644 packages/search/src/tokenize.ts create mode 100644 packages/search/test/file-index.spec.ts create mode 100644 packages/search/test/fixtures/source/sample.c create mode 100644 packages/search/test/fixtures/source/sample.cs create mode 100644 packages/search/test/fixtures/source/sample.generic create mode 100644 packages/search/test/fixtures/source/sample.go create mode 100644 packages/search/test/fixtures/source/sample.java create mode 100644 packages/search/test/fixtures/source/sample.kt create mode 100644 packages/search/test/fixtures/source/sample.php create mode 100644 packages/search/test/fixtures/source/sample.py create mode 100644 packages/search/test/fixtures/source/sample.rb create mode 100644 packages/search/test/fixtures/source/sample.rs create mode 100644 packages/search/test/fixtures/source/sample.ts create mode 100644 packages/search/test/project-files.spec.ts create mode 100644 packages/search/test/query-keywords.spec.ts create mode 100644 packages/search/test/snippet-index.spec.ts create mode 100644 packages/search/test/split-camelized.ts create mode 100644 packages/search/test/symbols.spec.ts create mode 100644 packages/search/test/tokenize.spec.ts create mode 100644 packages/search/tsconfig.json diff --git a/appmap.yml b/appmap.yml index 9e08e8b465..3f6954c9a2 100644 --- a/appmap.yml +++ b/appmap.yml @@ -11,6 +11,10 @@ packages: exclude: - node_modules - .yarn + - path: packages/search + exclude: + - node_modules + - .yarn - path: packages/client exclude: - node_modules diff --git a/packages/search/.eslintrc.js b/packages/search/.eslintrc.js new file mode 100644 index 0000000000..d98123a73e --- /dev/null +++ b/packages/search/.eslintrc.js @@ -0,0 +1,47 @@ +/* eslint-disable eslint-comments/disable-enable-pair */ +/* eslint-disable @typescript-eslint/no-var-requires */ +const path = require('path'); + +module.exports = { + plugins: ['@typescript-eslint', 'eslint-comments', 'jest', 'promise', 'unicorn'], + extends: [ + 'plugin:@typescript-eslint/recommended', + 'plugin:@typescript-eslint/recommended-requiring-type-checking', + 'plugin:eslint-comments/recommended', + 'plugin:jest/recommended', + 'plugin:promise/recommended', + 'prettier', + ], + env: { + node: true, + browser: true, + jest: true, + }, + parserOptions: { + project: path.join(__dirname, 'tsconfig.json'), + }, + root: true, + rules: { + 'no-param-reassign': ['error', { props: false }], + 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', + 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', + '@typescript-eslint/lines-between-class-members': 'off', + 'no-restricted-syntax': 'off', + 'import/no-cycle': 'off', + 'prettier/prettier': ['error'], + 'max-classes-per-file': 'off', + }, + overrides: [ + { + files: ['*.js'], + extends: ['eslint:recommended', 'prettier'], + parserOptions: { + project: path.join(__dirname, 'jsconfig.json'), + }, + plugins: ['prettier'], + rules: { + 'unicorn/prefer-module': 'off', + }, + }, + ], +}; diff --git a/packages/search/.gitignore b/packages/search/.gitignore new file mode 100644 index 0000000000..8bee2c9b8b --- /dev/null +++ b/packages/search/.gitignore @@ -0,0 +1,2 @@ +built + diff --git a/packages/search/jest.config.js b/packages/search/jest.config.js new file mode 100644 index 0000000000..01c9d384b6 --- /dev/null +++ b/packages/search/jest.config.js @@ -0,0 +1,11 @@ +/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testTimeout: parseInt(process.env.TEST_TIMEOUT, 10) || 5000, + silent: process.env.TEST_SILENT !== 'false', + restoreMocks: true, + // There are test cases that change the process working directory, and that does + // not work with multiple workers. + maxWorkers: 1, +}; diff --git a/packages/search/package.json b/packages/search/package.json new file mode 100644 index 0000000000..d0b2459214 --- /dev/null +++ b/packages/search/package.json @@ -0,0 +1,57 @@ +{ + "name": "@appland/search", + "version": "1.0.0", + "description": "", + "bin": "built/cli.js", + "publishConfig": { + "access": "public" + }, + "main": "built/index.js", + "types": "built/index.d.ts", + "files": [ + "built" + ], + "scripts": { + "lint": "eslint", + "lint:fix": "eslint --fix", + "test": "jest", + "build": "tsc", + "watch": "tsc --watch" + }, + "author": "AppLand, Inc", + "license": "Commons Clause + MIT", + "devDependencies": { + "@eslint/js": "~8.57.0", + "@types/better-sqlite3": "^7.6.9", + "@types/jest": "^29.5.4", + "@types/jest-sinon": "^1.0.2", + "@types/node": "^16", + "esbuild": "0.19.8", + "eslint": "^8.56.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-eslint-comments": "^3.2.0", + "eslint-plugin-import": "^2.22.1", + "eslint-plugin-jest": "^28.8.3", + "eslint-plugin-promise": "^7.1.0", + "eslint-plugin-unicorn": "latest", + "jest": "^29.7.0", + "jest-sinon": "^1.1.0", + "lint-staged": "^10.5.4", + "memfs": "^3.4.13", + "node-fetch": "2.6.7", + "prettier": "^2.7.1", + "sinon": "^11.1.2", + "ts-jest": "^29.0.5", + "ts-node": "^10.9.1", + "ts-sinon": "^2.0.2", + "tsc": "^2.0.3", + "type-fest": "^3.1.0", + "typescript": "^4.9.5", + "typescript-eslint": "^7.7.0" + }, + "dependencies": { + "better-sqlite3": "^9.5.0", + "package.json": "^2.0.1", + "yargs": "^17.7.2" + } +} diff --git a/packages/search/src/build-file-index.ts b/packages/search/src/build-file-index.ts new file mode 100644 index 0000000000..99e512ae4b --- /dev/null +++ b/packages/search/src/build-file-index.ts @@ -0,0 +1,71 @@ +import makeDebug from 'debug'; +import { join } from 'path'; + +import FileIndex from './file-index'; +import { ContentReader, readFileSafe } from './ioutil'; + +export type ListFn = (path: string) => Promise; + +export type FilterFn = (path: string) => PromiseLike; + +const debug = makeDebug('appmap:search:build-index'); + +export type Tokenizer = ( + content: string, + fileExtension: string +) => { symbols: string[]; words: string[] }; + +type Context = { + fileIndex: FileIndex; + baseDirectory: string; + listDirectory: ListFn; + fileFilter: FilterFn; + contentReader: ContentReader; + tokenizer: Tokenizer; +}; + +async function indexFile(context: Context, filePath: string) { + const fileContents = await context.contentReader(filePath); + if (!fileContents) return; + + const tokens = context.tokenizer(fileContents, filePath); + const symbols = tokens.symbols.join(' '); + const words = tokens.words.join(' '); + + context.fileIndex.indexFile(context.baseDirectory, filePath, symbols, words); +} + +async function indexDirectory(context: Context, directory: string) { + const dirContents = await context.listDirectory(directory); + if (!dirContents) return; + + for (const dirContentItem of dirContents) { + const filePath = join(directory, dirContentItem); + debug('Indexing: %s', filePath); + + if (await context.fileFilter(filePath)) { + indexFile(context, filePath); + } + } +} + +export default async function buildFileIndex( + fileIndex: FileIndex, + directories: string[], + listDirectory: ListFn, + fileFilter: FilterFn, + contentReader: ContentReader, + tokenizer: Tokenizer +): Promise { + for (const directory of directories) { + const context: Context = { + fileIndex, + baseDirectory: directory, + listDirectory, + fileFilter, + contentReader, + tokenizer, + }; + await indexDirectory(context, directory); + } +} diff --git a/packages/search/src/build-snippet-index.ts b/packages/search/src/build-snippet-index.ts new file mode 100644 index 0000000000..e4dd2f22f7 --- /dev/null +++ b/packages/search/src/build-snippet-index.ts @@ -0,0 +1,57 @@ +import { Tokenizer } from './build-file-index'; +import { ContentReader } from './ioutil'; +import SnippetIndex from './snippet-index'; +import { Splitter } from './splitter'; + +export type File = { + directory: string; + filePath: string; +}; + +type Context = { + snippetIndex: SnippetIndex; + contentReader: ContentReader; + splitter: Splitter; + tokenizer: Tokenizer; +}; + +async function indexFile(context: Context, file: File) { + const fileContent = await context.contentReader(file.filePath); + if (!fileContent) return; + + const extension = file.filePath.split('.').pop() || ''; + const chunks = await context.splitter(fileContent, extension); + + chunks.forEach((chunk, index) => { + const snippetId = `${file.filePath}:${index}`; + const { content, startLine, endLine } = chunk; + context.snippetIndex.indexSnippet( + snippetId, + file.directory, + file.filePath, + startLine, + endLine, + context.tokenizer(content, file.filePath).symbols.join(' '), + context.tokenizer(content, file.filePath).words.join(' '), + content + ); + }); +} + +export default async function buildSnippetIndex( + snippetIndex: SnippetIndex, + files: File[], + contentReader: ContentReader, + splitter: Splitter, + tokenizer: Tokenizer +) { + const context = { + snippetIndex, + contentReader, + splitter, + tokenizer, + }; + for (const file of files) { + await indexFile(context, file); + } +} diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts new file mode 100644 index 0000000000..dbe0baa54b --- /dev/null +++ b/packages/search/src/cli.ts @@ -0,0 +1,126 @@ +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import sqlite3 from 'better-sqlite3'; +import makeDebug from 'debug'; + +import { fileTokens } from './tokenize'; +import FileIndex from './file-index'; +import buildFileIndex from './build-file-index'; +import listProjectFiles from './project-files'; +import { isBinaryFile, isDataFile, isLargeFile } from './file-type'; +import SnippetIndex from './snippet-index'; +import buildSnippetIndex from './build-snippet-index'; +import { readFileSafe } from './ioutil'; +import { langchainSplitter } from './splitter'; +import assert from 'assert'; + +const debug = makeDebug('appmap:search:cli'); + +yargs(hideBin(process.argv)) + .command( + '* ', + 'Index directories and perform a search', + (yargs) => { + return yargs + .option('directories', { + alias: 'd', + type: 'array', + description: 'List of directories to index', + default: ['.'], + }) + .option('file-filter', { + type: 'string', + description: 'Regex pattern to filter files', + }) + .positional('query', { + describe: 'Search query', + type: 'string', + }) + .strict(); + }, + async (argv) => { + const { directories, query } = argv; + + let filterRE: RegExp | undefined; + if (argv.fileFilter) filterRE = new RegExp(argv.fileFilter); + + const fileFilter = async (path: string) => { + debug('Filtering: %s', path); + if (isBinaryFile(path)) { + debug('Skipping binary file: %s', path); + return false; + } + + const isData = isDataFile(path); + if (isData && (await isLargeFile(path))) { + debug('Skipping large data file: %s', path); + return false; + } + + if (!filterRE) return true; + + return !filterRE.test(path); + }; + + const db = new sqlite3(':memory:'); + const fileIndex = new FileIndex(db); + + await buildFileIndex( + fileIndex, + directories as string[], + listProjectFiles, + fileFilter, + readFileSafe, + fileTokens + ); + + const filePathAtMostThreeEntries = (filePath: string) => { + const parts = filePath.split('/'); + if (parts.length <= 3) return filePath; + + return `.../${parts.slice(-3).join('/')}`; + }; + + const printResult = (filePath: string, score: number) => + console.log('%s %s', filePathAtMostThreeEntries(filePath), score.toPrecision(3)); + + console.log('File search results'); + console.log('-------------------'); + const fileSearchResults = fileIndex.search(query as string); + for (const result of fileSearchResults) { + const { filePath, score } = result; + printResult(filePath, score); + } + + const splitter = langchainSplitter; + + const snippetIndex = new SnippetIndex(db); + await buildSnippetIndex(snippetIndex, fileSearchResults, readFileSafe, splitter, fileTokens); + + console.log(''); + console.log('Snippet search results'); + console.log('----------------------'); + + const isNullOrUndefined = (value: any) => value === null || value === undefined; + + const snippetSearchResults = snippetIndex.searchSnippets(query as string); + for (const result of snippetSearchResults) { + const { snippetId, filePath, startLine, endLine, score } = result; + printResult(snippetId, score); + + if (isNullOrUndefined(startLine) || isNullOrUndefined(endLine)) continue; + + const content = await readFileSafe(filePath); + if (!content) continue; + + assert(startLine !== undefined); + assert(endLine !== undefined); + + const lines = content.split('\n').slice(startLine - 1, endLine); + console.log(lines.map((l) => ` > ${l}`).join('\n')); + } + + db.close(); + } + ) + .help().argv; diff --git a/packages/search/src/file-index.ts b/packages/search/src/file-index.ts new file mode 100644 index 0000000000..9657b05f5e --- /dev/null +++ b/packages/search/src/file-index.ts @@ -0,0 +1,103 @@ +import sqlite3 from 'better-sqlite3'; + +const CREATE_TABLE_SQL = `CREATE VIRTUAL TABLE file_content USING fts5( + directory UNINDEXED, + file_path, + file_symbols, + file_words, + tokenize = 'porter unicode61' +)`; + +const CREATE_BOOST_TABLE_SQL = `CREATE TABLE file_boost ( + file_path TEXT PRIMARY KEY, + boost_factor REAL +)`; + +const INSERT_SQL = `INSERT INTO file_content (directory, file_path, file_symbols, file_words) +VALUES (?, ?, ?, ?)`; + +const UPDATE_BOOST_SQL = `INSERT OR REPLACE INTO file_boost (file_path, boost_factor) +VALUES (?, ?)`; + +const SEARCH_SQL = `SELECT + file_content.directory, + file_content.file_path, + (bm25(file_content, 1)*3.0 + bm25(file_content, 2)*2.0 + bm25(file_content, 3)*1.0) + * COALESCE(file_boost.boost_factor, 1.0) * -1 + AS score +FROM + file_content +LEFT JOIN + file_boost +ON + file_content.file_path = file_boost.file_path +WHERE + file_content MATCH ? +ORDER BY + score DESC +LIMIT + ? +`; + +type FileIndexRow = { + directory: string; + file_path: string; + score: number; +}; + +export type FileSearchResult = { + directory: string; + filePath: string; + score: number; +}; + +/** + * The FileIndex class provides an interface to interact with the SQLite search index. + * + * The primary responsibilities of this class include: + * 1. Indexing files by storing their directory paths, file paths, symbols (e.g., class names, method names), and + * general words in the database. Symbols are given more weight in the search results. + * 2. Boosting the relevance score of specific files based on external factors, such as AppMap trace data or error logs. + * 3. Performing search queries on the indexed files using full-text search with BM25 ranking. The search results are + * influenced by both the indexed content and any associated boost factors. + * + * The class uses two SQLite tables: + * - `file_content`: A virtual table that holds the file content and allows for full-text search using BM25 ranking. + * - `file_boost`: A table that stores boost factors for specific files to enhance their search relevance. + */ +export default class FileIndex { + #insert: sqlite3.Statement; + #updateBoost: sqlite3.Statement; + #search: sqlite3.Statement<[string, number]>; + + constructor(public database: sqlite3.Database) { + this.database.exec(CREATE_TABLE_SQL); + this.database.exec(CREATE_BOOST_TABLE_SQL); + this.database.pragma('journal_mode = OFF'); + this.database.pragma('synchronous = OFF'); + this.#insert = this.database.prepare(INSERT_SQL); + this.#updateBoost = this.database.prepare(UPDATE_BOOST_SQL); + this.#search = this.database.prepare(SEARCH_SQL); + } + + indexFile(directory: string, filePath: string, symbols: string, words: string): void { + this.#insert.run(directory, filePath, symbols, words); + } + + boostFile(filePath: string, boostFactor: number): void { + this.#updateBoost.run(filePath, boostFactor); + } + + search(query: string, limit = 10): FileSearchResult[] { + const rows = this.#search.all(query, limit) as FileIndexRow[]; + return rows.map((row) => ({ + directory: row.directory, + filePath: row.file_path, + score: row.score, + })); + } + + close() { + this.database.close(); + } +} diff --git a/packages/search/src/file-type.ts b/packages/search/src/file-type.ts new file mode 100644 index 0000000000..a4e044efb7 --- /dev/null +++ b/packages/search/src/file-type.ts @@ -0,0 +1,114 @@ +import { stat } from 'fs/promises'; +import makeDebug from 'debug'; + +const debug = makeDebug('appmap:search:file-type'); + +const BINARY_FILE_EXTENSIONS: string[] = [ + '7z', + 'aac', + 'avi', + 'bmp', + 'bz2', + 'class', + 'dll', + 'doc', + 'docx', + 'dylib', + 'ear', + 'exe', + 'eot', + 'flac', + 'flv', + 'gif', + 'gz', + 'ico', + 'jar', + 'jpeg', + 'jpg', + 'js.map', + 'min.js', + 'min.css', + 'mkv', + 'mo', + 'mov', + 'mp3', + 'mp4', + 'mpg', + 'odt', + 'odp', + 'ods', + 'ogg', + 'otf', + 'pdf', + 'po', + 'png', + 'ppt', + 'pptx', + 'pyc', + 'rar', + 'rtf', + 'so', + 'svg', + 'tar', + 'tiff', + 'ttf', + 'wav', + 'webm', + 'webp', + 'woff', + 'woff2', + 'wmv', + 'xls', + 'xlsx', + 'xz', + 'yarn.lock', + 'zip', +].map((ext) => '.' + ext); + +const DATA_FILE_EXTENSIONS: string[] = [ + 'cjs', + 'csv', + 'dat', + 'log', + 'json', + 'tsv', + 'yaml', + 'yml', + 'xml', +].map((ext) => '.' + ext); + +const DEFAULT_LARGE_FILE_THRESHOLD = 50_000; + +const largeFileThreshold = () => { + const value = process.env.APPMAP_LARGE_FILE; + if (value === undefined) return DEFAULT_LARGE_FILE_THRESHOLD; + return parseInt(value); +}; + +const statFileSafe = async (filePath: string): Promise => { + try { + const stats = await stat(filePath); + return stats.size; + } catch (error) { + debug(`Error reading file: %s`, filePath); + return undefined; + } +}; + +export const isLargeFile = async (fileName: string): Promise => { + const fileSize = await statFileSafe(fileName); + if (fileSize === undefined) return false; + + const isLarge = fileSize > largeFileThreshold(); + if (isLarge) debug('File %s is considered large due to size %d', fileName, fileSize); + + return fileSize > largeFileThreshold(); +}; + +export const isBinaryFile = (fileName: string): boolean => { + return BINARY_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext)); +}; + +export const isDataFile = (fileName: string): boolean => { + return DATA_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext)); +}; diff --git a/packages/search/src/git.ts b/packages/search/src/git.ts new file mode 100644 index 0000000000..0334c80ca6 --- /dev/null +++ b/packages/search/src/git.ts @@ -0,0 +1,172 @@ +import { exec as execCallback, spawn } from 'child_process'; +import { PathLike } from 'fs'; +import { promisify } from 'util'; + +const exec = promisify(execCallback); + +export enum GitState { + NotInstalled, // The git cli was not found. + NoRepository, // Git is installed but no repository was found. + Ok, // Git is installed, a repository is present. +} + +export const GitRepositoryEnvKeys = [ + 'GITHUB_REPOSITORY', // GitHub + 'CIRCLE_REPOSITORY_URL', // CircleCI + 'GIT_URL', // Jenkins + 'CI_REPOSITORY_URL', // GitLab +] as const; + +export const GitBranchEnvKeys = [ + 'GITHUB_REF_NAME', // GitHub + 'CIRCLE_BRANCH', // CircleCI + 'GIT_BRANCH', // Jenkins + 'TRAVIS_BRANCH', // TravisCI + 'CI_COMMIT_REF_NAME', // GitLab +] as const; + +export const GitCommitEnvKeys = [ + 'GITHUB_SHA', // GitHub + 'CIRCLE_SHA1', // CircleCI + 'GIT_COMMIT', // Jenkins + 'TRAVIS_COMMIT', // TravisCI + 'CI_COMMIT_SHA', // GitLab +] as const; + +class GitProperties { + static async contributors(sinceDaysAgo: number, cwd?: PathLike): Promise> { + const unixTimeNow = Math.floor(Number(new Date()) / 1000); + const unixTimeAgo = unixTimeNow - sinceDaysAgo * 24 * 60 * 60; + try { + const { stdout } = await exec( + [ + 'git', + cwd && `-C ${cwd.toString()}`, + '--no-pager', + 'log', + `--since=${unixTimeAgo}`, + '--format="%ae"', + ].join(' ') + ); + return [ + ...stdout + .trim() + .split('\n') + .reduce((acc, email) => { + acc.add(email); + return acc; + }, new Set()), + ]; + } catch { + return []; + } + } + + // Returns the repository URL, first by checking the environment, then by + // shelling out to git. + static async repository(cwd?: PathLike): Promise { + const envKey = GitRepositoryEnvKeys.find((key) => process.env[key]); + if (envKey) return process.env[envKey]; + + try { + const { stdout } = await exec( + ['git', cwd && `-C ${cwd.toString()}`, 'config', '--get', 'remote.origin.url'].join(' ') + ); + return stdout.trim(); + } catch { + return undefined; + } + } + + // Returns the branch, first by checking the environment, then by + // shelling out to git. + static async branch(cwd?: PathLike): Promise { + const envKey = GitBranchEnvKeys.find((key) => process.env[key]); + if (envKey) return process.env[envKey]; + + try { + const { stdout } = await exec( + ['git', cwd && `-C ${cwd.toString()}`, 'rev-parse', '--abbrev-ref', 'HEAD'].join(' ') + ); + return stdout.trim(); + } catch { + return undefined; + } + } + + // Returns the commit SHA, first by checking the environment, then by + // shelling out to git. + static async commit(cwd?: PathLike): Promise { + const envKey = GitCommitEnvKeys.find((key) => process.env[key]); + if (envKey) return process.env[envKey]; + + try { + const { stdout } = await exec( + ['git', cwd && `-C ${cwd.toString()}`, 'rev-parse', 'HEAD'].join(' ') + ); + return stdout.trim(); + } catch { + return undefined; + } + } + + static async state(cwd?: PathLike): Promise { + return new Promise((resolve) => { + try { + const commandProcess = spawn('git', ['status', '--porcelain'], { + shell: true, + cwd: cwd?.toString(), + stdio: 'ignore', + timeout: 2000, + }); + commandProcess.on('exit', (code) => { + switch (code) { + case 127: + return resolve(GitState.NotInstalled); + case 128: + return resolve(GitState.NoRepository); + default: + return resolve(GitState.Ok); + } + }); + commandProcess.on('error', () => resolve(GitState.NotInstalled)); + } catch { + resolve(GitState.NotInstalled); + } + }); + } + + static clearCache(): void { + gitCache.clear(); + } +} + +const gitCache = new Map(); +const noCacheList: Array = ['clearCache']; + +// GitProperties is available externally as Git. +// This export provides a simple caching layer around GitProperties to avoid +// excessive shelling out to git. +export const Git = new Proxy(GitProperties, { + get(target, prop) { + type TargetProp = keyof typeof target; + if ( + !noCacheList.includes(prop.toString() as TargetProp) && + typeof target[prop as TargetProp] === 'function' + ) { + return new Proxy(target[prop as TargetProp], { + apply(target, thisArg, argArray) { + const cacheKey = `${prop.toString()}(${JSON.stringify(argArray)})`; + if (gitCache.has(cacheKey)) { + return gitCache.get(cacheKey); + } + /* eslint-disable-next-line @typescript-eslint/ban-types */ + const result: unknown = Reflect.apply(target as Function, thisArg, argArray); + gitCache.set(cacheKey, result); + return result; + }, + }) as unknown; + } + return Reflect.get(target, prop) as unknown; + }, +}); diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts new file mode 100644 index 0000000000..ffe8ebb5f6 --- /dev/null +++ b/packages/search/src/index.ts @@ -0,0 +1,9 @@ +export { ContentReader, readFileSafe } from './ioutil'; +export { Splitter, langchainSplitter, wohleDocumentSplitter } from './splitter'; +export { ListFn, FilterFn, Tokenizer, default as buildFileIndex } from './build-file-index'; +export { File, default as buildSnippetIndex } from './build-snippet-index'; +export { default as SnippetIndex, SnippetSearchResult } from './snippet-index'; +export { default as FileIndex, FileSearchResult } from './file-index'; +export { default as listProjectFiles } from './project-files'; +export { isBinaryFile, isDataFile, isLargeFile } from './file-type'; +export { fileTokens } from './tokenize'; diff --git a/packages/search/src/ioutil.ts b/packages/search/src/ioutil.ts new file mode 100644 index 0000000000..0176171447 --- /dev/null +++ b/packages/search/src/ioutil.ts @@ -0,0 +1,15 @@ +import { readFileSync } from 'fs'; +import makeDebug from 'debug'; + +const debug = makeDebug('appmap:search:ioutil'); + +export type ContentReader = (filePath: string) => PromiseLike; + +export function readFileSafe(filePath: string): PromiseLike { + try { + return Promise.resolve(readFileSync(filePath, 'utf8')); + } catch (error) { + debug(`Error reading file: %s`, filePath); + return Promise.resolve(undefined); + } +} diff --git a/packages/search/src/project-files.ts b/packages/search/src/project-files.ts new file mode 100644 index 0000000000..23d220eed8 --- /dev/null +++ b/packages/search/src/project-files.ts @@ -0,0 +1,110 @@ +import { promisify } from 'util'; +import makeDebug from 'debug'; +import { exec as execCb } from 'node:child_process'; + +import { Git, GitState } from './git'; +import assert from 'assert'; +import { readdir } from 'fs/promises'; +import { join, relative } from 'path'; + +const debug = makeDebug('appmap:search:project-files'); +const exec = promisify(execCb); + +export default async function listProjectFiles(directory: string): Promise { + const gitState = await Git.state(directory); + debug(`Git state: %s`, gitState); + return gitState === GitState.Ok + ? await listGitProjectFiles(directory) + : await listLikelyProjectFiles(directory); +} + +// Run git ls-files and git status to get a list of all git-managed files. By doing it this way, +// we automatically apply any .gitignore rules. +export async function listGitProjectFiles(directory: string): Promise { + const lsFiles = async (): Promise => { + try { + const { stdout } = await exec('git ls-files', { + cwd: directory, + maxBuffer: 1024 ** 2 * 20, // 20 MB + }); + + debug(stdout); + + return stdout.split('\n').filter(Boolean); + } catch (e) { + debug('`git ls-files` failed: %s', e); + return []; + } + }; + + const statusFiles = async (): Promise => { + try { + const { stdout } = await exec('git status --porcelain', { + cwd: directory, + maxBuffer: 1024 ** 2 * 20, // 20 MB + }); + + debug(stdout); + + return stdout + .split('\n') + .map((line) => { + // git status --porcelain output starts with 3 characters: staged status, unstaged status, + // and a space. + return line.slice(3); + }) + .filter(Boolean); + } catch (e) { + debug('`git status --porcelain` failed: %s', e); + return []; + } + }; + + return Array.from(new Set([...(await lsFiles()), ...(await statusFiles())])); +} + +const IGNORE_DIRECTORIES = new Set([ + '.git', + '.venv', + '.yarn', + 'node_modules', + 'vendor', + 'build', + 'built', + 'dist', + 'out', + 'target', + 'tmp', + 'venv', +]); + +// Produce a modest-sized listing of files in the project. +// Ignore a standard list of binary file extensions and directories that tend to be full of +// non-source files. +export async function listLikelyProjectFiles(directory: string): Promise { + const files = new Array(); + + const ignoreDirectory = (dir: string) => IGNORE_DIRECTORIES.has(dir); + + // Perform a breadth-first traversal of a directory, collecting all non-binary files and + // applying the directory ignore list. + const processDir = async (dir: string) => { + const queue = [dir]; + while (queue.length > 0) { + const currentDir = queue.shift(); + assert(currentDir, 'queue should not be empty'); + + const entries = await readdir(currentDir, { withFileTypes: true }); + for (const entry of entries) { + const path = join(currentDir, entry.name); + if (entry.isDirectory()) { + if (!ignoreDirectory(entry.name)) queue.push(path); + } else files.push(relative(dir, path)); + } + } + }; + + await processDir(directory); + + return files; +} diff --git a/packages/search/src/query-keywords.ts b/packages/search/src/query-keywords.ts new file mode 100644 index 0000000000..e3835840a8 --- /dev/null +++ b/packages/search/src/query-keywords.ts @@ -0,0 +1,76 @@ +import { splitCamelized } from './split-camelized'; + +const STOP_WORDS = new Set([ + 'a', + 'an', + 'and', + 'are', + 'as', + 'at', + 'be', + 'by', + 'code', + 'for', + 'from', + 'has', + 'he', + 'in', + 'is', + 'it', + 'its', + 'of', + 'on', + 'over', + 'that', + 'the', + 'to', + 'was', + 'were', + 'will', + 'with', + 'without', +]); + +/** + * Replace non-alphanumeric characters with spaces, then split the keyword on spaces. + * So in effect, words with non-alphanumeric characters become multiple words. + * Allow dash and underscore as delimeters. + */ +const sanitizeKeyword = (keyword: string): string[] => + keyword.replace(/[^\p{L}\p{N}\-_]/gu, ' ').split(' '); + +/** + * Extract keywords from a string or an array of strings. The extraction process includes the following steps: + * + * - Remove non-alphanumeric characters and split the keyword on spaces. + * - Split camelized words. + * - Remove stop words. + */ +export default function queryKeywords(words: undefined | string | string[]): string[] { + if (!words) return []; + + const wordsArray = Array.isArray(words) ? words : [words]; + if (wordsArray.length === 0) return []; + + return wordsArray + .map((word) => sanitizeKeyword(word || '')) + .flat() + .filter(Boolean) + .map((word): string[] => { + const camelized = splitCamelized(word) + .split(/[\s\-_]/) + .map((word) => word.toLowerCase()); + // Return each of the component words, and also return each pair of adjacent words as a single word. + const result = new Array(); + for (let i = 0; i < camelized.length; i++) { + result.push(camelized[i]); + if (i > 0) result.push([camelized[i - 1] + camelized[i]].join('')); + } + return result; + }) + .flat() + .map((str) => str.trim()) + .filter(Boolean) + .filter((str) => str.length >= 2) + .filter((str) => !STOP_WORDS.has(str)); +} diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts new file mode 100644 index 0000000000..81986af2ab --- /dev/null +++ b/packages/search/src/snippet-index.ts @@ -0,0 +1,118 @@ +import sqlite3 from 'better-sqlite3'; +import { warn } from 'console'; + +const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5( + snippet_id UNINDEXED, + directory UNINDEXED, + file_path, + start_line UNINDEXED, + end_line UNINDEXED, + file_symbols, + file_words, + content UNINDEXED, + tokenize = 'porter unicode61' +)`; + +const CREATE_SNIPPET_BOOST_TABLE_SQL = `CREATE TABLE snippet_boost ( + snippet_id TEXT PRIMARY KEY, + boost_factor REAL +)`; + +const INSERT_SNIPPET_SQL = `INSERT INTO snippet_content +(snippet_id, directory, file_path, start_line, end_line, file_symbols, file_words, content) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; + +const UPDATE_SNIPPET_BOOST_SQL = `INSERT OR REPLACE INTO snippet_boost +(snippet_id, boost_factor) +VALUES (?, ?)`; + +const SEARCH_SNIPPET_SQL = `SELECT + snippet_content.directory, + snippet_content.file_path, + snippet_content.start_line, + snippet_content.end_line, + snippet_content.snippet_id, + snippet_content.content, + (bm25(snippet_content, 1)*3.0 + bm25(snippet_content, 2)*2.0 + bm25(snippet_content, 3)*1.0) + * COALESCE(snippet_boost.boost_factor, 1.0) * -1 + AS score +FROM + snippet_content +LEFT JOIN + snippet_boost +ON + snippet_content.snippet_id = snippet_boost.snippet_id +WHERE + snippet_content MATCH ? +ORDER BY + score DESC +LIMIT ?`; + +export type SnippetSearchResult = { + snippetId: string; + directory: string; + filePath: string; + startLine: number | undefined; + endLine: number | undefined; + score: number; + content: string; +}; + +export default class SnippetIndex { + #insertSnippet: sqlite3.Statement; + #updateSnippetBoost: sqlite3.Statement; + #searchSnippet: sqlite3.Statement<[string, number]>; + + constructor(public database: sqlite3.Database) { + this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL); + this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL); + this.database.pragma('journal_mode = OFF'); + this.database.pragma('synchronous = OFF'); + this.#insertSnippet = this.database.prepare(INSERT_SNIPPET_SQL); + this.#updateSnippetBoost = this.database.prepare(UPDATE_SNIPPET_BOOST_SQL); + this.#searchSnippet = this.database.prepare(SEARCH_SNIPPET_SQL); + } + + indexSnippet( + snippetId: string, + directory: string, + filePath: string, + startLine: number | undefined, + endLine: number | undefined, + symbols: string, + words: string, + content: string + ): void { + this.#insertSnippet.run( + snippetId, + directory, + filePath, + startLine, + endLine, + symbols, + words, + content + ); + } + + boostSnippet(snippetId: string, boostFactor: number): void { + this.#updateSnippetBoost.run(snippetId, boostFactor); + } + + searchSnippets(query: string, limit = 10): SnippetSearchResult[] { + const rows = this.#searchSnippet.all(query, limit) as any[]; + return rows.map((row) => ({ + directory: row.directory, + snippetId: row.snippet_id, + filePath: row.file_path, + startLine: row.start_line, + endLine: row.end_line, + score: row.score, + content: row.content, + })); + } + + close() { + this.database.close(); + } +} diff --git a/packages/search/src/split-camelized.ts b/packages/search/src/split-camelized.ts new file mode 100644 index 0000000000..f3708d2e8b --- /dev/null +++ b/packages/search/src/split-camelized.ts @@ -0,0 +1,71 @@ +import { log } from 'console'; + +export const LOG_CAMELIZED_TO_RAW = process.env.APPMAP_LOG_CAMELIZED_TO_RAW === 'true'; +export const CAMELIZED_TO_RAW = new Map(); + +/** + * Split a camelized word into a new word that is separated by a given separator. + */ +// Derived from https://raw.githubusercontent.com/sindresorhus/decamelize/main/index.js +// MIT License +// Copyright (c) Sindre Sorhus sindresorhus@gmail.com (https://sindresorhus.com) +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +export function splitCamelized( + text: string, + { separator = ' ', preserveConsecutiveUppercase = true } = {} +): string { + const handlePreserveConsecutiveUppercase = (decamelized: string, separator: string): string => { + // Lowercase all single uppercase characters. As we + // want to preserve uppercase sequences, we cannot + // simply lowercase the separated string at the end. + // `data_For_USACounties` → `data_for_USACounties` + decamelized = decamelized.replace( + /((? $0.toLowerCase() + ); + + // Remaining uppercase sequences will be separated from lowercase sequences. + // `data_For_USACounties` → `data_for_USA_counties` + return decamelized.replace( + /(\p{Uppercase_Letter}+)(\p{Uppercase_Letter}\p{Lowercase_Letter}+)/gu, + (_, $1, $2) => $1 + separator + $2.toLowerCase() + ); + }; + + // Checking the second character is done later on. Therefore process shorter strings here. + if (text.length < 2) { + return preserveConsecutiveUppercase ? text : text.toLowerCase(); + } + + const replacement = `$1${separator}$2`; + + // Split lowercase sequences followed by uppercase character. + // `dataForUSACounties` → `data_For_USACounties` + // `myURLstring → `my_URLstring` + const decamelized = text.replace( + /([\p{Lowercase_Letter}\d])(\p{Uppercase_Letter})/gu, + replacement + ); + + let result: string; + if (preserveConsecutiveUppercase) { + result = handlePreserveConsecutiveUppercase(decamelized, separator); + } else { + // Split multiple uppercase characters followed by one or more lowercase characters. + // `my_URLstring` → `my_ur_lstring` + result = decamelized + .replace(/(\p{Uppercase_Letter})(\p{Uppercase_Letter}\p{Lowercase_Letter}+)/gu, replacement) + .toLowerCase(); + } + + if (LOG_CAMELIZED_TO_RAW) { + if (!CAMELIZED_TO_RAW.has(result)) { + log(`[splitCamelized] ${text} → ${result}`); + CAMELIZED_TO_RAW.set(result, text); + } + } + + return result; +} diff --git a/packages/search/src/splitter.ts b/packages/search/src/splitter.ts new file mode 100644 index 0000000000..2188ba77d4 --- /dev/null +++ b/packages/search/src/splitter.ts @@ -0,0 +1,76 @@ +import { + RecursiveCharacterTextSplitter, + SupportedTextSplitterLanguage, +} from 'langchain/text_splitter'; +import makeDebug from 'debug'; +import { warn } from 'console'; + +export type Chunk = { + content: string; + startLine?: number; + endLine?: number; +}; + +export type Splitter = (content: string, fileExtension: string) => PromiseLike; + +const debug = makeDebug('appmap:search:splitter'); + +const TEXT_SPLITTER_LANGUAGE_EXTENSIONS: Record = { + cpp: ['cpp', 'h', 'hpp', 'c', 'cc', 'cxx', 'hxx'], + go: ['go'], + java: ['java', 'jsp', 'jspx'], + js: ['js', 'ts', 'mjs', 'jsx', 'tsx', 'vue', 'svelte'], + php: ['php'], + proto: ['proto'], + python: ['py'], + rst: ['rst'], + ruby: ['rb', 'haml', 'erb'], + rust: ['rs'], + scala: ['scala'], + swift: ['swift'], + markdown: ['md'], + latex: ['tex'], + html: ['html'], + sol: ['sol'], +}; + +export function wohleDocumentSplitter(content: string, fileExtension: string): Promise { + return Promise.resolve([ + { + content, + startLine: 1, + endLine: content.split('\n').length, + }, + ]); +} + +export async function langchainSplitter(content: string, fileExtension: string): Promise { + const language = Object.keys(TEXT_SPLITTER_LANGUAGE_EXTENSIONS).find((language) => + TEXT_SPLITTER_LANGUAGE_EXTENSIONS[language as SupportedTextSplitterLanguage].includes( + fileExtension + ) + ) as SupportedTextSplitterLanguage | undefined; + let splitter: RecursiveCharacterTextSplitter; + if (language) { + splitter = RecursiveCharacterTextSplitter.fromLanguage(language); + } else { + debug('No language found for extension: %s', fileExtension); + splitter = new RecursiveCharacterTextSplitter(); + } + const documents = await splitter.createDocuments([content]); + + // metadata includes: + // { loc: { lines: { from: 1, to: 14 } } } + + return documents.map((doc) => { + const lines = doc.metadata?.loc?.lines; + const result: Chunk = { + content: doc.pageContent, + }; + if (lines) { + result.startLine = lines.from; + result.endLine = lines.to; + } + return result; + }); +} diff --git a/packages/search/src/tokenize.ts b/packages/search/src/tokenize.ts new file mode 100644 index 0000000000..caf9a49e14 --- /dev/null +++ b/packages/search/src/tokenize.ts @@ -0,0 +1,104 @@ +import queryKeywords from './query-keywords'; +import makeDebug from 'debug'; + +const debug = makeDebug('appmap:search:tokenize'); + +export const SymbolRegexes: Record = { + cs: /(((interface|class|enum|struct)\s+(?\w+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, + cpp: /(((struct|enum|union|class)\s+(?\w+)\s*?\{)|(}\s*?(?\w+)\s*?;)|((\s|^)(?!try|catch|if|while|do|for|switch)(?[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, + rs: /(struct|enum|union|trait|type|fn)\s+(?[\w\p{L}]+)/gu, + go: /((type\s+(?[\w\p{L}]+))|(func\s+?(\(.*?\)\s*?)?(?[\w\p{L}]+)\s*?\())/gu, + rb: /(((class|module)\s+(?\w+))|(def\s+?(?\w+)))/g, + py: /(class|def)\s+(?\w+)/g, + java: /(((class|@?interface|enum)\s+(?[\w$]+))|((\s|^)(?!try|catch|if|while|do|for|switch)(?[\w$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, + ts: /(((class|interface|enum|type|function)\s+(?[#$\w\p{L}]+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?[#$\w\p{L}]+)\s*?\([^;)]*?\)[\w\s<>[\].:\n]*?\{)|((?[#$\w\p{L}]+)\s*?(=|:)\s*?\(.*?\)\s*?=>))/gu, + kt: /(((class|typealias)\s+(?[\w_]+))|(fun\s+?(<.+?>\s+)?(.*?\.)?(?\w+)))/g, + php: /(class|trait|function)\s+(?[\w_$]+)/g, +}; + +const genericSymbolRegex = + /(((^|\s)(?!using|try|catch|if|while|do|for|switch)(?[#$\w\p{L}~]+)\s*?\(([^;)])*?\)[\w\s<>[\].:\n]*?\{)|(^(?!.*?(?:#|\/\/|"|')).*?(interface|class|enum|struct|union|trait|type(alias|def)?|fu?nc?(tion)?|module|def)\s+?(?[#$\w\p{L}]+))|((?[#$\w\p{L}~]+)\s*?=\s*?[\w\s<>[\].:\n]*?\{))/gmu; + +// Define aliases for common file extensions +['js', 'jsx', 'ts', 'tsx', 'vue', 'svelte'].forEach((ext) => { + SymbolRegexes[ext] = SymbolRegexes.ts; +}); + +['c', 'cc', 'cxx', 'h', 'hpp', 'cpp', 'hxx'].forEach((ext) => { + SymbolRegexes[ext] = SymbolRegexes.cpp; +}); + +function getMatches(source: string, regex: RegExp): string[] { + const results: string[] = []; + const matches = source.matchAll(regex); + + for (const match of matches) { + const { groups } = match; + const symbol = groups?.symbol1 ?? groups?.symbol2 ?? groups?.symbol3; + + if (symbol) results.push(symbol); + } + + return results; +} + +export function symbols(content: string, fileExtension: string, allowGeneric = true): string[] { + let regex = allowGeneric ? genericSymbolRegex : undefined; + if (fileExtension && fileExtension in SymbolRegexes) { + regex = SymbolRegexes[fileExtension]; + } + + if (regex) { + return getMatches(content, regex); + } + + return []; +} + +export function words(content: string): string[] { + return content.match(/\b\w+\b/g) ?? []; +} + +type FileTokens = { + symbols: string[]; + words: string[]; +}; + +export function fileTokens( + content: string, + fileExtension: string, + enableGenericSymbolParsing = true +): FileTokens { + if (enableGenericSymbolParsing) + debug('Using generic symbol parsing for file extension: %s', fileExtension); + + const symbolList = queryKeywords( + symbols(content, fileExtension, enableGenericSymbolParsing) + ).sort(); + const wordList = queryKeywords(words(content)).sort(); + + // Iterate through words, with a corresponding pointer to symbols. + // If the word at the word index does not match the symbol at the symbol index, + // add the word to the output. Otherwise, advance both pointers. Repeat + // until all words have been traversed. + const filteredWordList = new Array(); + let symbolIndex = 0; + let wordIndex = 0; + const collectWord = () => { + const word = wordList[wordIndex]; + const symbol = symbolList[symbolIndex]; + if (word === symbol) { + symbolIndex += 1; + } else { + filteredWordList.push(word); + } + wordIndex += 1; + }; + + while (wordIndex < wordList.length) collectWord(); + + return { + symbols: symbolList, + words: filteredWordList, + }; +} diff --git a/packages/search/test/file-index.spec.ts b/packages/search/test/file-index.spec.ts new file mode 100644 index 0000000000..f5a1d6c1c2 --- /dev/null +++ b/packages/search/test/file-index.spec.ts @@ -0,0 +1,49 @@ +import { strict as assert } from 'assert'; +import sqlite3 from 'better-sqlite3'; +import FileIndex, { FileSearchResult } from '../src/file-index'; + +describe('FileIndex', () => { + let db: sqlite3.Database; + let index: FileIndex; + const directory = 'src'; + + beforeEach(() => { + db = new sqlite3(':memory:'); + index = new FileIndex(db); + }); + + afterEach(() => { + if (index) index.close(); + }); + + it('should insert and search a file', () => { + index.indexFile(directory, 'test.txt', 'symbol1', 'word1'); + const results = index.search('symbol1'); + assert.equal(results.length, 1); + assert.equal(results[0].filePath, 'test.txt'); + }); + + it('should update the boost factor of a file', () => { + index.indexFile(directory, 'test2.txt', 'symbol2', 'word2'); + index.boostFile('test2.txt', 2.0); + const results = index.search('symbol2'); + expect(results.map((r: FileSearchResult) => r.directory)).toEqual([directory]); + expect(results.map((r: FileSearchResult) => r.filePath)).toEqual(['test2.txt']); + expect(results[0].score).toBeGreaterThan(0); + }); + + it('should return results ordered by score', () => { + index.indexFile(directory, 'test3.txt', 'symbol1 symbol3', 'word1 word3'); + index.indexFile(directory, 'test4.txt', 'symbol2 symbol3', 'word1 word4'); + index.boostFile('test4.txt', 2.0); + + let results = index.search('word1'); + expect(results.map((r: FileSearchResult) => r.filePath)).toEqual(['test4.txt', 'test3.txt']); + + results = index.search('symbol3'); + expect(results.map((r: FileSearchResult) => r.filePath)).toEqual(['test4.txt', 'test3.txt']); + + results = index.search('symbol2'); + expect(results.map((r: FileSearchResult) => r.filePath)).toEqual(['test4.txt']); + }); +}); diff --git a/packages/search/test/fixtures/source/sample.c b/packages/search/test/fixtures/source/sample.c new file mode 100644 index 0000000000..6180c8a0a9 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.c @@ -0,0 +1,30 @@ +#define MODE_DEFAULT 0 +#define MODE_NONE (MODE_DEFAULT - 1) +#define MODE_SUPER (MODE_DEFAULT + 1) + +typedef struct { + int a; + int b; +} Point; + +struct MyStruct { + int a; + int b; +}; + +struct MyOtherStruct +{ +}; + +void foo() +{ + +} + +int main() { + Point p; + foo(); + p.a = 1; + p.b = 2; + return p.a + p.b; +} diff --git a/packages/search/test/fixtures/source/sample.cs b/packages/search/test/fixtures/source/sample.cs new file mode 100644 index 0000000000..7999b063e1 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.cs @@ -0,0 +1,75 @@ +public class ClassOne { } + +public class ClassTwo : IInterface { } + +public partial class ClassThree { } + +class ClassFour +{ + +} + +public struct StructOne { } + +struct StructTwo +{ +} + +readonly public struct StructThree : IInterface { } + +interface IOne { } + +public interface ITwo : ISomethingElse +{ + +}; + +private interface IThree { } + +namespace MyNamespace +{ + interface IFour + { + } +} + +enum Example { One, Two, Three } + +enum Season +{ + Spring, + Summer, + Autumn, + Winter +} + +enum ErrorCode : ushort +{ + None = 0, + Unknown = 1, + + ConnectionLost = 100, + OutlierReading = 200 +} + +class ClassWithMethods +{ + ClassWithMethods() { } + ~ClassWithMethods() { } + public void MethodOne() { } + public static void MethodTwo() { } + ISomething MethodThree() + { + FunctionCall() + } + ISomething MethodFour() + { + } +} + + +#if (DEBUG) +Console.WriteLine("Hello World"); +#else +Console.WriteLine("Goodbye World"); +#endif diff --git a/packages/search/test/fixtures/source/sample.generic b/packages/search/test/fixtures/source/sample.generic new file mode 100644 index 0000000000..6dead1eb50 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.generic @@ -0,0 +1,103 @@ +// dart +// via https://dart.dev/language + +int fibonacci(int n) { + if (n == 0 || n == 1) return n; + return fibonacci(n - 1) + fibonacci(n - 2); +} + +class Spacecraft { + String name; + DateTime? launchDate; + + // Read-only non-final property + int? get launchYear => launchDate?.year; + + // Constructor, with syntactic sugar for assignment to members. + Spacecraft(this.name, this.launchDate) { + // Initialization code goes here. + } + + // Named constructor that forwards to the default one. + Spacecraft.unlaunched(String name) : this(name, null); + + // Method. + void describe() { + print('Spacecraft: $name'); + // Type promotion doesn't work on getters. + var launchDate = this.launchDate; + if (launchDate != null) { + int years = DateTime.now().difference(launchDate).inDays ~/ 365; + print('Launched: $launchYear ($years years ago)'); + } else { + print('Unlaunched'); + } + } +} + +enum PlanetType { terrestrial, gas, ice } + +// swift +// via https://docs.swift.org/swift-book/documentation/the-swift-programming-language + +struct Resolution { + var width = 0 + var height = 0 +} + +class VideoMode { + var resolution = Resolution() + var interlaced = false + var frameRate = 0.0 + var name: String? +} + +class Counter { + var count = 0 + func increment() { + count += 1 + } + func increment(by amount: Int) { + count += amount + } + func reset() { + count = 0 + } +} + +// gdscript +// via https://docs.godotengine.org/en/stable/tutorials/scripting/gdscript/gdscript_basics.html + +func something(p1, p2): + super(p1, p2) + + +# It's also possible to call another function in the super class: +func other_something(p1, p2): + super.something(p1, p2) + + +# Inner class +class Something: + var a = 10 + + +# Constructor +func _init(): + print("Constructed!") + var lv = Something.new() + print(lv.a) + + +// misc + +# fn dont_include_me +# class CommentedOut {} + +// fn dont_include_me_either +// class CommentedOutToo {} + +// qc +void() perform_action = { + +} diff --git a/packages/search/test/fixtures/source/sample.go b/packages/search/test/fixtures/source/sample.go new file mode 100644 index 0000000000..826bc2d088 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.go @@ -0,0 +1,36 @@ +type Locker interface { + Lock() + Unlock() +} + +type MyInterface interface { + // TODO: Interface methods are not captured + // This symbol will not be reported + MyMethod() +} + +type LockerImpl struct { +} + +func (l *LockerImpl) Lock() { +} + +func (l *LockerImpl) Unlock() { +} + +type Reader interface { + Read(p []byte) (n int, err error) +} + +type ReadWriter interface { + Reader // includes methods of Reader in ReadWriter's method set + Writer // includes methods of Writer in ReadWriter's method set +} + +func main() { + fmt.Println("Hello, World!") +} + +func unicodeβ(s string) string { + return s +} diff --git a/packages/search/test/fixtures/source/sample.java b/packages/search/test/fixtures/source/sample.java new file mode 100644 index 0000000000..2e447d0772 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.java @@ -0,0 +1,21 @@ +public enum SampleEnum { + A, B, C +} + +public @interface SampleAnnotation { +} + +public interface ISample { + public void sampleMethod(); +} + +public class Sample { + public static void main(String[] args) { + System.out.println("Hello, world!"); + } +} + +public void performUserOperation(User user, + Operation op) { + super.performUserOperation(user, op); +} diff --git a/packages/search/test/fixtures/source/sample.kt b/packages/search/test/fixtures/source/sample.kt new file mode 100644 index 0000000000..46178e153c --- /dev/null +++ b/packages/search/test/fixtures/source/sample.kt @@ -0,0 +1,23 @@ +fun main(args: Array) { + println(args.contentToString()) +} + +class Rectangle(val height: Double, val length: Double) { + val perimeter = (height + length) * 2 +} + +typealias Predicate = (T) -> Boolean + +enum class Color(val value: Int) { + RED(1), + GREEN(2), + BLUE(3) +} + +fun List.filter(predicate: Predicate): List { + return this.filter(predicate) +} + +fun > sort(list: List): List { + return list.sorted() +} diff --git a/packages/search/test/fixtures/source/sample.php b/packages/search/test/fixtures/source/sample.php new file mode 100644 index 0000000000..09eeb74cf9 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.php @@ -0,0 +1,18 @@ + diff --git a/packages/search/test/fixtures/source/sample.py b/packages/search/test/fixtures/source/sample.py new file mode 100644 index 0000000000..3a1d7e1f04 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.py @@ -0,0 +1,9 @@ +def some_function(): + print("Hello, world!") + +class MyClass: + def __init__(self, name): + self.name = name + + def say_hello(self): + print(f"Hello, {self.name}!") diff --git a/packages/search/test/fixtures/source/sample.rb b/packages/search/test/fixtures/source/sample.rb new file mode 100644 index 0000000000..fa2aa56a07 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.rb @@ -0,0 +1,16 @@ +class MyClass + def my_method + puts 'Hello world' + end +end + +module MyModule + def my_module_method + puts 'Hello world' + end +end + +def some_function(name) + calling_a_method + puts "Hello #{name}" +end diff --git a/packages/search/test/fixtures/source/sample.rs b/packages/search/test/fixtures/source/sample.rs new file mode 100644 index 0000000000..2bd44c8707 --- /dev/null +++ b/packages/search/test/fixtures/source/sample.rs @@ -0,0 +1,33 @@ +struct Foo { + bar: String, +} + +impl Foo { + fn new(bar: String) -> Self { + Self { bar } + } +} + +#[repr(C)] +union MyUnion { + f1: u32, + f2: f32, +} + +trait MyTrait { + fn my_function(&self); +} + +type Point = (u8, u8); + +fn main() { + let foo = Foo { + bar: "Hello, world!".to_string(), + }; + + println!("{}", foo.bar); +} + +fn Москва() -> Option { + None +} diff --git a/packages/search/test/fixtures/source/sample.ts b/packages/search/test/fixtures/source/sample.ts new file mode 100644 index 0000000000..fc5984338b --- /dev/null +++ b/packages/search/test/fixtures/source/sample.ts @@ -0,0 +1,46 @@ +class MyClass { + public readonly name: string; + + constructor() { + this.name = 'MyClass'; + } + + #myClassMethod() { + return 'myClassMethod'; + } +} + +function myFunction() { + return 'myFunction'; +} + +const myOtherFunction = () => 'myOtherFunction'; + +type MyType = { + name: string; +}; + +interface MyInterface { + name: string; +} + +enum MyEnum { + A = 'A', + B = 'B', + C = 'C', +} + +const myObject = { + myObjectMethod: () => {}, +}; + +// prettier-ignore +function newLineBraces() +{ +} + +// prettier-ignore +const $lineBreak = () => + { + + }; diff --git a/packages/search/test/project-files.spec.ts b/packages/search/test/project-files.spec.ts new file mode 100644 index 0000000000..f29d909402 --- /dev/null +++ b/packages/search/test/project-files.spec.ts @@ -0,0 +1,102 @@ +import * as fs from 'fs'; +import * as fsp from 'fs/promises'; +import { join } from 'path'; +import * as childProcess from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; + +import listProjectFiles, { listGitProjectFiles } from '../src/project-files'; + +jest.mock('fs/promises', () => ({ + readdir: jest.fn(), +})); +jest.mock('node:child_process'); + +describe('listProjectFiles', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('lists all files in a directory', async () => { + const mockFiles = [ + { name: 'index.js', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, + { name: 'logo.png', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, + { name: 'utils', isFile: () => false, isDirectory: () => true } as unknown as fs.Dirent, + // Pretend utils directory contains a single file + { name: 'helper.js', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, + ]; + + const baseDir = join('fake', 'directory'); + jest.mocked(fsp.readdir).mockImplementation((dir: fs.PathLike) => { + if (dir.toString() === join(baseDir, 'utils')) return Promise.resolve([mockFiles[3]]); + else if (dir.toString() === baseDir) return Promise.resolve(mockFiles.slice(0, 3)); + else return Promise.resolve([]); + }); + + const files = await listProjectFiles(baseDir); + expect(files).toContain('index.js'); + expect(files).toContain(join('utils', 'helper.js')); + expect(fsp.readdir).toHaveBeenCalledTimes(2); // baseDir + utils + }); + + it('ignores directories specified in IGNORE_DIRECTORIES', async () => { + const mockDirectories = [ + { + name: 'node_modules', + isFile: () => false, + isDirectory: () => true, + } as unknown as fs.Dirent, + { name: 'src', isFile: () => false, isDirectory: () => true } as unknown as fs.Dirent, + ]; + + const directoriesRead = new Array(); + const baseDir = join('fake', 'directory'); + jest.mocked(fsp.readdir).mockImplementation((dir: fs.PathLike) => { + directoriesRead.push(dir.toString()); + + if (dir.toString() === baseDir) return Promise.resolve(mockDirectories); + else return Promise.resolve([]); + }); + + const files = await listProjectFiles(baseDir); + expect(files).toEqual([]); + expect(directoriesRead).toEqual([baseDir, join(baseDir, 'src')]); + }); +}); + +describe('listGitProjectFiles', () => { + const mockDirectory = '/mock/directory'; + + beforeEach(() => { + // Clear all mocks before each test + jest.clearAllMocks(); + // Setup default return values for mocked functions + jest.mocked(childProcess.exec).mockImplementation((cmd: string, _options, callback) => { + const cb = callback as (_: unknown, __: unknown) => void; + if (cmd.includes('ls-files')) { + cb(null, { stdout: 'file1.js' }); + } else if (cmd.includes('status --porcelain')) { + cb(null, { + stdout: ` +?? newFile.ts +M stagedFile.js + M unstagedFile.js +`, + }); + } else { + cb(null, { stdout: '' }); + } + + return {} as ChildProcess; + }); + }); + + it('includes git-managed files, including modified files, as well as untracked', async () => { + const files = await listGitProjectFiles(mockDirectory); + + expect(files.length).toBe(4); + expect(files).toContain('file1.js'); + expect(files).toContain('newFile.ts'); + expect(files).toContain('stagedFile.js'); + expect(files).toContain('unstagedFile.js'); + }); +}); diff --git a/packages/search/test/query-keywords.spec.ts b/packages/search/test/query-keywords.spec.ts new file mode 100644 index 0000000000..1f129edc0e --- /dev/null +++ b/packages/search/test/query-keywords.spec.ts @@ -0,0 +1,83 @@ +import queryKeywords from '../src/query-keywords'; + +describe('queryKeywords', () => { + it('should sanitize and split keywords correctly', () => { + const input = 'Hello_World! Welcome to the universe.'; + expect(queryKeywords(input)).toEqual(['hello', 'world', 'helloworld', 'welcome', 'universe']); + }); + + it('should filter out stop words', () => { + const input = 'the quick brown fox jumps over the lazy dog'; + expect(queryKeywords(input)).toEqual(['quick', 'brown', 'fox', 'jumps', 'lazy', 'dog']); + }); + + it('should handle under_score words', () => { + const input = 'hello_world this_is_a_test'; + expect(queryKeywords(input)).toEqual([ + 'hello', + 'world', + 'helloworld', + 'this', + 'thisis', + 'isa', + 'test', + 'atest', + ]); + }); + + it('should handle camelCased words', () => { + const input = 'helloWorld thisIsATest'; + expect(queryKeywords(input)).toEqual([ + 'hello', + 'world', + 'helloworld', + 'this', + 'thisis', + 'isa', + 'test', + 'atest', + ]); + }); + + it('should handle array input', () => { + const input = ['hello', 'world']; + expect(queryKeywords(input)).toEqual(['hello', 'world']); + }); + + it('should return empty array for invalid input', () => { + expect(queryKeywords(undefined)).toEqual([]); + expect(queryKeywords('')).toEqual([]); + }); + + it('should only return words with length >= 2', () => { + const input = 'a an i am'; + expect(queryKeywords(input)).toEqual(['am']); + }); + + it('should split camelCased words with consecutive uppercase letters', () => { + const input = 'dataForUSACounties'; + expect(queryKeywords(input)).toEqual([ + 'data', + 'datafor', + 'usa', + 'forusa', + 'counties', + 'usacounties', + ]); + }); + + it('should handle strings with multiple camelizations', () => { + const input = 'XMLHttpRequest and HTTPResponseCode'; + expect(queryKeywords(input)).toEqual([ + 'xml', + 'http', + 'xmlhttp', + 'request', + 'httprequest', + 'http', + 'response', + 'httpresponse', + 'responsecode', + ]); + }); +}); diff --git a/packages/search/test/snippet-index.spec.ts b/packages/search/test/snippet-index.spec.ts new file mode 100644 index 0000000000..bbf256722f --- /dev/null +++ b/packages/search/test/snippet-index.spec.ts @@ -0,0 +1,84 @@ +import { strict as assert } from 'assert'; +import sqlite3 from 'better-sqlite3'; + +import SnippetIndex from '../src/snippet-index'; + +describe('SnippetIndex', () => { + let db: sqlite3.Database; + let index: SnippetIndex; + const directory = 'src'; + + beforeEach(() => { + db = new sqlite3(':memory:'); + index = new SnippetIndex(db); + }); + + afterEach(() => { + if (index) index.close(); + }); + + it('should insert and search a snippet', () => { + const content = 'symbol1 word1'; + index.indexSnippet('snippet1', directory, 'test.txt', 1, 10, 'symbol1', 'word1', content); + const results = index.searchSnippets('symbol1'); + assert.equal(results.length, 1); + assert.equal(results[0].snippetId, 'snippet1'); + assert.equal(results[0].content, content); + }); + + it('should update the boost factor of a snippet', () => { + const content = 'symbol2 word2'; + index.indexSnippet('snippet2', directory, 'test2.txt', 11, 20, 'symbol2', 'word2', content); + index.boostSnippet('snippet2', 2.0); + const results = index.searchSnippets('symbol2'); + assert.equal(results.length, 1); + assert.equal(results[0].snippetId, 'snippet2'); + }); + + it('should return results ordered by score', () => { + index.indexSnippet( + 'snippet3', + directory, + 'test3.txt', + 21, + 30, + 'symbol1 symbol3', + 'word1 word3', + 'symbol1 word1 symbol3 word3' + ); + index.indexSnippet( + 'snippet4', + directory, + 'test4.txt', + 31, + 40, + 'symbol2 symbol3', + 'word1 word4', + 'symbol2 word1 symbol3 word4' + ); + + let results = index.searchSnippets('word1 OR word4'); + assert.equal(results.length, 2); + assert.equal(results[0].snippetId, 'snippet4'); + assert.equal(results[1].snippetId, 'snippet3'); + + const unboostedScore = results[1].score; + + index.boostSnippet('snippet3', 2.0); + + results = index.searchSnippets('word1 OR word4'); + assert.equal(results.length, 2); + assert.equal(results[0].snippetId, 'snippet3'); + assert.equal(results[1].snippetId, 'snippet4'); + + const boostedScore = results[0].score; + const scoreMultiple = boostedScore / unboostedScore; + expect(scoreMultiple).toBeGreaterThan(1.99); + expect(scoreMultiple).toBeLessThan(2.01); + + results = index.searchSnippets('symbol3'); + assert.equal(results.length, 2); + assert.equal(results[0].snippetId, 'snippet3'); + assert.equal(results[1].snippetId, 'snippet4'); + }); +}); diff --git a/packages/search/test/split-camelized.ts b/packages/search/test/split-camelized.ts new file mode 100644 index 0000000000..aea06875dd --- /dev/null +++ b/packages/search/test/split-camelized.ts @@ -0,0 +1,31 @@ +import { splitCamelized } from '../src/split-camelized'; + +describe('splitCamelized', () => { + it('should split camelized strings correctly', () => { + const cases = [ + { text: 'dataForUSACounties', expected: 'data for USA counties' }, + { text: 'myURLstring', expected: 'my URL string' }, + { text: 'simpleTest', expected: 'simple test' }, + { text: 'CAPSTest', expected: 'caps test' }, + ]; + + cases.forEach(({ text, expected }) => { + const result = splitCamelized(text, { separator: ' ', preserveConsecutiveUppercase: true }); + expect(result).toBe(expected); + }); + }); + + it('should handle short strings correctly', () => { + expect(splitCamelized('A', { separator: '_', preserveConsecutiveUppercase: false })).toBe('a'); + expect(splitCamelized('A', { separator: '-', preserveConsecutiveUppercase: true })).toBe('A'); + }); + + it('should respect preserveConsecutiveUppercase flag', () => { + expect( + splitCamelized('dataForUSACounties', { separator: ' ', preserveConsecutiveUppercase: false }) + ).toBe('data for usa counties'); + expect(splitCamelized('URLtest', { separator: '_', preserveConsecutiveUppercase: true })).toBe( + 'URL_test' + ); + }); +}); diff --git a/packages/search/test/symbols.spec.ts b/packages/search/test/symbols.spec.ts new file mode 100644 index 0000000000..b9554beffe --- /dev/null +++ b/packages/search/test/symbols.spec.ts @@ -0,0 +1,195 @@ +import { join } from 'path'; +import { symbols } from '../src/tokenize'; +import { readFileSync } from 'fs'; + +describe('querySymbols', () => { + const fileSymbols = (srcPath: string, allowGeneric = true) => { + const fileExtension = srcPath.split('.').pop() || ''; + const content = readFileSync(srcPath, 'utf-8'); + return symbols(content, fileExtension, allowGeneric); + }; + + describe('csharp', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.cs'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + [ + 'ClassOne', + 'ClassTwo', + 'ClassThree', + 'ClassFour', + 'StructOne', + 'StructTwo', + 'StructThree', + 'IOne', + 'ITwo', + 'IThree', + 'IFour', + 'Example', + 'Season', + 'ErrorCode', + 'ClassWithMethods', + 'ClassWithMethods', + '~ClassWithMethods', + 'MethodOne', + 'MethodTwo', + 'MethodThree', + 'MethodFour', + ].sort() + ); + }); + }); + + describe('c/cpp', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.c'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['foo', 'main', 'MyStruct', 'MyOtherStruct', 'Point'].sort() + ); + }); + }); + + describe('rust', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.rs'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['Foo', 'new', 'MyUnion', 'MyTrait', 'my_function', 'Point', 'main', 'Москва'].sort() + ); + }); + }); + + describe('go', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.go'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + [ + 'Locker', + 'MyInterface', + 'LockerImpl', + 'Lock', + 'Unlock', + 'Reader', + 'ReadWriter', + 'main', + 'unicodeβ', + ].sort() + ); + }); + }); + + describe('ruby', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.rb'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['MyClass', 'MyModule', 'my_method', 'my_module_method', 'some_function'].sort() + ); + }); + }); + + describe('python', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.py'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['MyClass', '__init__', 'say_hello', 'some_function'].sort() + ); + }); + }); + + describe('java', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.java'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + [ + 'SampleEnum', + 'SampleAnnotation', + 'ISample', + 'Sample', + 'main', + 'performUserOperation', + ].sort() + ); + }); + }); + + describe('javascript/typescript', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.ts'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + [ + 'MyClass', + 'constructor', + '#myClassMethod', + 'myFunction', + 'myOtherFunction', + 'MyType', + 'MyInterface', + 'MyEnum', + 'myObjectMethod', + 'newLineBraces', + '$lineBreak', + ].sort() + ); + }); + }); + + describe('kotlin', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.kt'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['main', 'Rectangle', 'Predicate', 'Color', 'filter', 'sort'].sort() + ); + }); + }); + + describe('php', () => { + it('identifies symbols', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.php'); + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + ['MyClass', 'myMethod', 'Talk', 'talk', 'myFunction'].sort() + ); + }); + }); + + describe('generic', () => { + const srcPath = join(__dirname, 'fixtures/source/sample.generic'); + + it('identifies symbols', () => { + const symbols = fileSymbols(srcPath); + expect(symbols.sort()).toStrictEqual( + [ + 'fibonacci', + 'Spacecraft', + 'Spacecraft', + 'describe', + 'PlanetType', + 'Resolution', + 'VideoMode', + 'Counter', + 'increment', + 'increment', + 'reset', + 'Something', + '_init', + 'other_something', + 'something', + 'perform_action', + ].sort() + ); + }); + + it('does not run if allowGeneric is false', () => { + const symbols = fileSymbols(srcPath, false); + expect(symbols).toStrictEqual([]); + }); + }); +}); diff --git a/packages/search/test/tokenize.spec.ts b/packages/search/test/tokenize.spec.ts new file mode 100644 index 0000000000..4e32df4438 --- /dev/null +++ b/packages/search/test/tokenize.spec.ts @@ -0,0 +1,37 @@ +import { symbols, words, fileTokens } from '../src/tokenize'; + +describe('FileTokens', () => { + const content = ` + class Example { + public void method1() { } + public void method2() { } + } + `; + + const fileExtension = 'java'; + + it('should extract symbols from the content', () => { + const result = symbols(content, fileExtension); + expect(result).toEqual(['Example', 'method1', 'method2']); + }); + + it('should extract words from the content', () => { + const result = words(content); + expect(result).toEqual([ + 'class', + 'Example', + 'public', + 'void', + 'method1', + 'public', + 'void', + 'method2', + ]); + }); + + it('should extract file tokens (symbols and words) from the content', () => { + const result = fileTokens(content, fileExtension); + expect(result.symbols).toEqual(['example', 'method1', 'method2']); + expect(result.words).toEqual(['class', 'public', 'public', 'void', 'void']); + }); +}); diff --git a/packages/search/tsconfig.json b/packages/search/tsconfig.json new file mode 100644 index 0000000000..6fed487f49 --- /dev/null +++ b/packages/search/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "declaration": true, + "outDir": "built", + "resolveJsonModule": true, + "target": "es2021", + "types": ["node", "jest"] + }, + "include": ["src"] +} diff --git a/yarn.lock b/yarn.lock index 2c3938d5b4..714c9ab774 100644 --- a/yarn.lock +++ b/yarn.lock @@ -152,6 +152,7 @@ __metadata: "@appland/openapi": "workspace:^1.8.0" "@appland/rpc": "workspace:^1.13.0" "@appland/scanner": "workspace:^1.86.0" + "@appland/search": "workspace:^1.0.0" "@appland/sequence-diagram": "workspace:^1.12.0" "@craftamap/esbuild-plugin-html": ^0.4.0 "@eslint/js": ~8.57.0 @@ -585,6 +586,45 @@ __metadata: languageName: unknown linkType: soft +"@appland/search@workspace:^1.0.0, @appland/search@workspace:packages/search": + version: 0.0.0-use.local + resolution: "@appland/search@workspace:packages/search" + dependencies: + "@eslint/js": ~8.57.0 + "@types/better-sqlite3": ^7.6.9 + "@types/jest": ^29.5.4 + "@types/jest-sinon": ^1.0.2 + "@types/node": ^16 + better-sqlite3: ^9.5.0 + esbuild: 0.19.8 + eslint: ^8.56.0 + eslint-config-prettier: ^8.3.0 + eslint-plugin-eslint-comments: ^3.2.0 + eslint-plugin-import: ^2.22.1 + eslint-plugin-jest: ^28.8.3 + eslint-plugin-promise: ^7.1.0 + eslint-plugin-unicorn: latest + jest: ^29.7.0 + jest-sinon: ^1.1.0 + lint-staged: ^10.5.4 + memfs: ^3.4.13 + node-fetch: 2.6.7 + package.json: ^2.0.1 + prettier: ^2.7.1 + sinon: ^11.1.2 + ts-jest: ^29.0.5 + ts-node: ^10.9.1 + ts-sinon: ^2.0.2 + tsc: ^2.0.3 + type-fest: ^3.1.0 + typescript: ^4.9.5 + typescript-eslint: ^7.7.0 + yargs: ^17.7.2 + bin: + search: built/cli.js + languageName: unknown + linkType: soft + "@appland/sequence-diagram@^1.11.0, @appland/sequence-diagram@workspace:^1.12.0, @appland/sequence-diagram@workspace:packages/sequence-diagram": version: 0.0.0-use.local resolution: "@appland/sequence-diagram@workspace:packages/sequence-diagram" @@ -1851,6 +1891,13 @@ __metadata: languageName: node linkType: hard +"@babel/helper-validator-identifier@npm:^7.24.7": + version: 7.25.7 + resolution: "@babel/helper-validator-identifier@npm:7.25.7" + checksum: 062f55208deead4876eb474dc6fd55155c9eada8d0a505434de3b9aa06c34195562e0f3142b22a08793a38d740238efa2fe00ff42956cdcb8ac03f0b6c542247 + languageName: node + linkType: hard + "@babel/helper-validator-option@npm:^7.16.7": version: 7.16.7 resolution: "@babel/helper-validator-option@npm:7.16.7" @@ -11252,6 +11299,15 @@ __metadata: languageName: node linkType: hard +"@types/keyv@npm:^3.1.1": + version: 3.1.4 + resolution: "@types/keyv@npm:3.1.4" + dependencies: + "@types/node": "*" + checksum: e009a2bfb50e90ca9b7c6e8f648f8464067271fd99116f881073fa6fa76dc8d0133181dd65e6614d5fb1220d671d67b0124aef7d97dc02d7e342ab143a47779d + languageName: node + linkType: hard + "@types/lodash@npm:^4.14.167": version: 4.14.191 resolution: "@types/lodash@npm:4.14.191" @@ -11515,6 +11571,15 @@ __metadata: languageName: node linkType: hard +"@types/responselike@npm:^1.0.0": + version: 1.0.3 + resolution: "@types/responselike@npm:1.0.3" + dependencies: + "@types/node": "*" + checksum: 6ac4b35723429b11b117e813c7acc42c3af8b5554caaf1fc750404c1ae59f9b7376bc69b9e9e194a5a97357a597c2228b7173d317320f0360d617b6425212f58 + languageName: node + linkType: hard + "@types/retry@npm:0.12.0": version: 0.12.0 resolution: "@types/retry@npm:0.12.0" @@ -13881,6 +13946,15 @@ __metadata: languageName: node linkType: hard +"abs@npm:^1.2.1": + version: 1.3.14 + resolution: "abs@npm:1.3.14" + dependencies: + ul: ^5.0.0 + checksum: af5fc49949f0694f458b7849e8ab9f10d2f6a2e1a93b3ac5cb25d62ad331d9f4d37a72fcbc1a84591fdf90fa3fffba8bf6de60dd483dd1af2200166f3436b3d3 + languageName: node + linkType: hard + "accepts@npm:~1.3.4, accepts@npm:~1.3.5, accepts@npm:~1.3.7, accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -16295,6 +16369,20 @@ __metadata: languageName: node linkType: hard +"browserslist@npm:^4.23.3": + version: 4.24.0 + resolution: "browserslist@npm:4.24.0" + dependencies: + caniuse-lite: ^1.0.30001663 + electron-to-chromium: ^1.5.28 + node-releases: ^2.0.18 + update-browserslist-db: ^1.1.0 + bin: + browserslist: cli.js + checksum: de200d3eb8d6ed819dad99719099a28fb6ebeb88016a5ac42fbdc11607e910c236a84ca1b0bbf232477d4b88ab64e8ab6aa67557cdd40a73ca9c2834f92ccce0 + languageName: node + linkType: hard + "bs-logger@npm:0.x, bs-logger@npm:^0.2.6": version: 0.2.6 resolution: "bs-logger@npm:0.2.6" @@ -16400,6 +16488,13 @@ __metadata: languageName: node linkType: hard +"builtin-modules@npm:^3.3.0": + version: 3.3.0 + resolution: "builtin-modules@npm:3.3.0" + checksum: db021755d7ed8be048f25668fe2117620861ef6703ea2c65ed2779c9e3636d5c3b82325bd912244293959ff3ae303afa3471f6a15bf5060c103e4cc3a839749d + languageName: node + linkType: hard + "builtin-status-codes@npm:^3.0.0": version: 3.0.0 resolution: "builtin-status-codes@npm:3.0.0" @@ -16776,6 +16871,13 @@ __metadata: languageName: node linkType: hard +"caniuse-lite@npm:^1.0.30001663": + version: 1.0.30001669 + resolution: "caniuse-lite@npm:1.0.30001669" + checksum: 8ed0c69d0c6aa3b1cbc5ba4e5f5330943e7b7165e257f6955b8b73f043d07ad922265261f2b54d9bbaf02886bbdba5e6f5b16662310a13f91f17035af3212de1 + languageName: node + linkType: hard + "capture-exit@npm:^2.0.0": version: 2.0.0 resolution: "capture-exit@npm:2.0.0" @@ -16785,6 +16887,13 @@ __metadata: languageName: node linkType: hard +"capture-stack-trace@npm:^1.0.0": + version: 1.0.2 + resolution: "capture-stack-trace@npm:1.0.2" + checksum: 13295e8176e8de74bcbe0e4fd938bed9eb4204b4cc200210ff46df91cb20b69e86f6ef42f408a59454f8b62e567ef0ee6ee5b5e7e16e686668bc77f2741542b4 + languageName: node + linkType: hard + "cardinal@npm:^2.1.1": version: 2.1.1 resolution: "cardinal@npm:2.1.1" @@ -17050,6 +17159,13 @@ __metadata: languageName: node linkType: hard +"ci-info@npm:^4.0.0": + version: 4.0.0 + resolution: "ci-info@npm:4.0.0" + checksum: 122fe41c5eb8d0b5fa0ab6fd674c5ddcf2dc59766528b062a0144ff0d913cfb210ef925ec52110e7c2a7f4e603d5f0e8b91cfe68867e196e9212fa0b94d0a08a + languageName: node + linkType: hard + "cidr-regex@npm:^3.1.1": version: 3.1.1 resolution: "cidr-regex@npm:3.1.1" @@ -17980,6 +18096,15 @@ __metadata: languageName: node linkType: hard +"core-js-compat@npm:^3.38.1": + version: 3.38.1 + resolution: "core-js-compat@npm:3.38.1" + dependencies: + browserslist: ^4.23.3 + checksum: a0a5673bcd59f588f0cd0b59cdacd4712b82909738a87406d334dd412eb3d273ae72b275bdd8e8fef63fca9ef12b42ed651be139c7c44c8a1acb423c8906992e + languageName: node + linkType: hard + "core-js@npm:^2.4.0, core-js@npm:^2.5.0": version: 2.6.12 resolution: "core-js@npm:2.6.12" @@ -18135,6 +18260,15 @@ __metadata: languageName: node linkType: hard +"create-error-class@npm:^3.0.1": + version: 3.0.2 + resolution: "create-error-class@npm:3.0.2" + dependencies: + capture-stack-trace: ^1.0.0 + checksum: 7254a6f96002d3226d3c1fec952473398761eb4fb12624c5dce6ed0017cdfad6de39b29aa7139680d7dcf416c25f2f308efda6eb6d9b7123f829b19ef8271511 + languageName: node + linkType: hard + "create-hash@npm:^1.1.0, create-hash@npm:^1.1.2, create-hash@npm:^1.2.0": version: 1.2.0 resolution: "create-hash@npm:1.2.0" @@ -19486,6 +19620,15 @@ __metadata: languageName: node linkType: hard +"deffy@npm:^2.2.1, deffy@npm:^2.2.2": + version: 2.2.4 + resolution: "deffy@npm:2.2.4" + dependencies: + typpy: ^2.0.0 + checksum: a06f44306c676d5fb663120610060a3abc48c745200f306a11acc2936095f901b170018fa5ffedbc1ebddf43b83467dc11b90b9ec5fdd232f970d85d66515543 + languageName: node + linkType: hard + "define-data-property@npm:^1.0.1, define-data-property@npm:^1.1.1": version: 1.1.1 resolution: "define-data-property@npm:1.1.1" @@ -20174,7 +20317,7 @@ __metadata: languageName: node linkType: hard -"duplexer2@npm:~0.1.0": +"duplexer2@npm:^0.1.4, duplexer2@npm:~0.1.0": version: 0.1.4 resolution: "duplexer2@npm:0.1.4" dependencies: @@ -20320,6 +20463,13 @@ __metadata: languageName: node linkType: hard +"electron-to-chromium@npm:^1.5.28": + version: 1.5.41 + resolution: "electron-to-chromium@npm:1.5.41" + checksum: 942cc53beabeb0647598d432155e2c21bed0de3dfd46576112aeed4157ea59543875c8a99038c5b05e8843fb3b91f14278ed4ea2bf4943845b26456ec20d2c9b + languageName: node + linkType: hard + "elkjs@npm:^0.8.2": version: 0.8.2 resolution: "elkjs@npm:0.8.2" @@ -20547,6 +20697,15 @@ __metadata: languageName: node linkType: hard +"err@npm:^1.1.1": + version: 1.1.1 + resolution: "err@npm:1.1.1" + dependencies: + typpy: ^2.2.0 + checksum: e9b8b5226724e32c7fdf2a14c1380093e130e1bfa4e15f6886ffd7239f0a51b8733da7b8790f01a6681f641a97838d0537e18d553528ab52245a05e6dcf75fe7 + languageName: node + linkType: hard + "errno@npm:^0.1.1, errno@npm:^0.1.3, errno@npm:~0.1.7": version: 0.1.8 resolution: "errno@npm:0.1.8" @@ -20558,7 +20717,7 @@ __metadata: languageName: node linkType: hard -"error-ex@npm:^1.3.1": +"error-ex@npm:^1.2.0, error-ex@npm:^1.3.1": version: 1.3.2 resolution: "error-ex@npm:1.3.2" dependencies: @@ -21225,6 +21384,13 @@ __metadata: languageName: node linkType: hard +"escalade@npm:^3.2.0": + version: 3.2.0 + resolution: "escalade@npm:3.2.0" + checksum: 47b029c83de01b0d17ad99ed766347b974b0d628e848de404018f3abee728e987da0d2d370ad4574aa3d5b5bfc368754fd085d69a30f8e75903486ec4b5b709e + languageName: node + linkType: hard + "escape-html@npm:~1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" @@ -21607,6 +21773,15 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-promise@npm:^7.1.0": + version: 7.1.0 + resolution: "eslint-plugin-promise@npm:7.1.0" + peerDependencies: + eslint: ^7.0.0 || ^8.0.0 || ^9.0.0 + checksum: 6b4d28cbb33847c732262aee8a1784cdfc37754e91611ac0c86bfd8dd07fbba966a953a212ae49f91de1f4a31b28d65153935b0f183ce14647b31451706f722b + languageName: node + linkType: hard + "eslint-plugin-react@npm:^7.30.1": version: 7.32.2 resolution: "eslint-plugin-react@npm:7.32.2" @@ -21697,6 +21872,32 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-unicorn@npm:latest": + version: 56.0.0 + resolution: "eslint-plugin-unicorn@npm:56.0.0" + dependencies: + "@babel/helper-validator-identifier": ^7.24.7 + "@eslint-community/eslint-utils": ^4.4.0 + ci-info: ^4.0.0 + clean-regexp: ^1.0.0 + core-js-compat: ^3.38.1 + esquery: ^1.6.0 + globals: ^15.9.0 + indent-string: ^4.0.0 + is-builtin-module: ^3.2.1 + jsesc: ^3.0.2 + pluralize: ^8.0.0 + read-pkg-up: ^7.0.1 + regexp-tree: ^0.1.27 + regjsparser: ^0.10.0 + semver: ^7.6.3 + strip-indent: ^3.0.0 + peerDependencies: + eslint: ">=8.56.0" + checksum: 4bc05fac0b49523de08452ebf0238d2e7f7b5790c234b22b9d74035efa656f01d79f7adcff8e5b075e9813e3cae9c36ba8e1956c1b12449c64c7af9c6a36726d + languageName: node + linkType: hard + "eslint-plugin-vue@npm:^9.9.0": version: 9.9.0 resolution: "eslint-plugin-vue@npm:9.9.0" @@ -22041,6 +22242,15 @@ __metadata: languageName: node linkType: hard +"esquery@npm:^1.6.0": + version: 1.6.0 + resolution: "esquery@npm:1.6.0" + dependencies: + estraverse: ^5.1.0 + checksum: 08ec4fe446d9ab27186da274d979558557fbdbbd10968fa9758552482720c54152a5640e08b9009e5a30706b66aba510692054d4129d32d0e12e05bbc0b96fb2 + languageName: node + linkType: hard + "esrecurse@npm:^4.1.0, esrecurse@npm:^4.3.0": version: 4.3.0 resolution: "esrecurse@npm:4.3.0" @@ -22167,6 +22377,16 @@ __metadata: languageName: node linkType: hard +"exec-limiter@npm:^3.0.0": + version: 3.2.13 + resolution: "exec-limiter@npm:3.2.13" + dependencies: + limit-it: ^3.0.0 + typpy: ^2.1.0 + checksum: 61c4d7e222bf5d062e9bf9a067cc9346c6d16cce935e5e4ddaa0c2fbcbda9b214545f04aeabc3bb902a17b0b340b6710f4e7ead4999ed255764c028074a1b3c7 + languageName: node + linkType: hard + "exec-sh@npm:^0.3.2": version: 0.3.6 resolution: "exec-sh@npm:0.3.6" @@ -23468,6 +23688,15 @@ __metadata: languageName: node linkType: hard +"function.name@npm:^1.0.3": + version: 1.0.13 + resolution: "function.name@npm:1.0.13" + dependencies: + noop6: ^1.0.1 + checksum: 376bd4247cacffb50ae425c64e99c42fd10a875197e09db7e65002a5bc0f539608b03f1ff7e27234516cd5b3bbfd9d9d8e912fa9aaeec52d3b422d501f290fff + languageName: node + linkType: hard + "function.prototype.name@npm:^1.1.0, function.prototype.name@npm:^1.1.5": version: 1.1.5 resolution: "function.prototype.name@npm:1.1.5" @@ -23776,6 +24005,51 @@ __metadata: languageName: node linkType: hard +"git-package-json@npm:^1.4.0": + version: 1.4.10 + resolution: "git-package-json@npm:1.4.10" + dependencies: + deffy: ^2.2.1 + err: ^1.1.1 + gry: ^5.0.0 + normalize-package-data: ^2.3.5 + oargv: ^3.4.1 + one-by-one: ^3.1.0 + r-json: ^1.2.1 + r-package-json: ^1.0.0 + tmp: 0.0.28 + checksum: 69c16f42e9dfbfd60daf5f7cf33642b2cf4ebf90ffac452b4eacf12c3d9f1728da1c0d771348f6adccd7f4c4a3a7f271e19fbfbb81e16c4096143dfe7ad7df4a + languageName: node + linkType: hard + +"git-source@npm:^1.1.0": + version: 1.1.10 + resolution: "git-source@npm:1.1.10" + dependencies: + git-url-parse: ^5.0.1 + checksum: d3efbe1b32994c82827b43fa5e5de2503bf5e7f44c048b27514d6617f6e0119f4f609c899ad261c63661749867938767bff0a5180c2a3e6193ba386e497e17be + languageName: node + linkType: hard + +"git-up@npm:^1.0.0": + version: 1.2.1 + resolution: "git-up@npm:1.2.1" + dependencies: + is-ssh: ^1.0.0 + parse-url: ^1.0.0 + checksum: 45a939df02e949a1608858b472b50d7165a17521ded288365d0c5b0eb8fa157ba14a6ceaeda316ebdcf4c1c455bbd037a86cc48432723d8c094404d881357b0d + languageName: node + linkType: hard + +"git-url-parse@npm:^5.0.1": + version: 5.0.1 + resolution: "git-url-parse@npm:5.0.1" + dependencies: + git-up: ^1.0.0 + checksum: 081c37feac708e93601ab8b0c6c5b465984dd62334e0f5121ca95f4d4d22fa668f97ade78ed5fb42f3b637ab3c60785eae2e60402e5c356fae97292791d425b1 + languageName: node + linkType: hard + "gitconfiglocal@npm:^2.1.0": version: 2.1.0 resolution: "gitconfiglocal@npm:2.1.0" @@ -24026,6 +24300,13 @@ __metadata: languageName: node linkType: hard +"globals@npm:^15.9.0": + version: 15.11.0 + resolution: "globals@npm:15.11.0" + checksum: ef32d5ef987f3d4b47fc2e389a0b235f6a46f605160c4e405722fd7b576106ca407cb867e66fd1e0fc43b631800e2e2e71847f37691026d813f96f40339da702 + languageName: node + linkType: hard + "globals@npm:^9.18.0": version: 9.18.0 resolution: "globals@npm:9.18.0" @@ -24117,6 +24398,29 @@ __metadata: languageName: node linkType: hard +"got@npm:^5.0.0": + version: 5.7.1 + resolution: "got@npm:5.7.1" + dependencies: + create-error-class: ^3.0.1 + duplexer2: ^0.1.4 + is-redirect: ^1.0.0 + is-retry-allowed: ^1.0.0 + is-stream: ^1.0.0 + lowercase-keys: ^1.0.0 + node-status-codes: ^1.0.0 + object-assign: ^4.0.1 + parse-json: ^2.1.0 + pinkie-promise: ^2.0.0 + read-all-stream: ^3.0.0 + readable-stream: ^2.0.5 + timed-out: ^3.0.0 + unzip-response: ^1.0.2 + url-parse-lax: ^1.0.0 + checksum: bd8e0d0b28e27bc34b81ce98a7ff116b2dec77ac70dda9cf624000eca303ebff51a7c00c1113bede3038afa2e865a61e6a5a6d2c645c5adf68ac8ed5e5b8d064 + languageName: node + linkType: hard + "graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.15, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.4, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.9 resolution: "graceful-fs@npm:4.2.9" @@ -24168,6 +24472,18 @@ __metadata: languageName: node linkType: hard +"gry@npm:^5.0.0": + version: 5.0.8 + resolution: "gry@npm:5.0.8" + dependencies: + abs: ^1.2.1 + exec-limiter: ^3.0.0 + one-by-one: ^3.0.0 + ul: ^5.0.0 + checksum: 969eb7a98f8e5f290e2fab9e689a2cef02099ef92abceafdc9d7074356e34e2ab275a56f148867ead9d7db79b6dc7458a8208b805c4a9dbd43b5a406d5402d7c + languageName: node + linkType: hard + "gunzip-maybe@npm:^1.4.2": version: 1.4.2 resolution: "gunzip-maybe@npm:1.4.2" @@ -25564,6 +25880,15 @@ __metadata: languageName: node linkType: hard +"is-builtin-module@npm:^3.2.1": + version: 3.2.1 + resolution: "is-builtin-module@npm:3.2.1" + dependencies: + builtin-modules: ^3.3.0 + checksum: e8f0ffc19a98240bda9c7ada84d846486365af88d14616e737d280d378695c8c448a621dcafc8332dbf0fcd0a17b0763b845400709963fa9151ddffece90ae88 + languageName: node + linkType: hard + "is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.4": version: 1.2.4 resolution: "is-callable@npm:1.2.4" @@ -26083,6 +26408,13 @@ __metadata: languageName: node linkType: hard +"is-redirect@npm:^1.0.0": + version: 1.0.0 + resolution: "is-redirect@npm:1.0.0" + checksum: 25dd3d9943f57ef0f29d28e2d9deda8288e0c7098ddc65abec3364ced9a6491ea06cfaf5110c61fc40ec1fde706b73cee5d171f85278edbf4e409b85725bfea7 + languageName: node + linkType: hard + "is-reference@npm:^1.1.2, is-reference@npm:^1.2.1": version: 1.2.1 resolution: "is-reference@npm:1.2.1" @@ -26116,6 +26448,13 @@ __metadata: languageName: node linkType: hard +"is-retry-allowed@npm:^1.0.0": + version: 1.2.0 + resolution: "is-retry-allowed@npm:1.2.0" + checksum: 50d700a89ae31926b1c91b3eb0104dbceeac8790d8b80d02f5c76d9a75c2056f1bb24b5268a8a018dead606bddf116b2262e5ac07401eb8b8783b266ed22558d + languageName: node + linkType: hard + "is-set@npm:^2.0.1, is-set@npm:^2.0.2": version: 2.0.2 resolution: "is-set@npm:2.0.2" @@ -26139,7 +26478,16 @@ __metadata: languageName: node linkType: hard -"is-stream@npm:^1.1.0": +"is-ssh@npm:^1.0.0, is-ssh@npm:^1.3.0": + version: 1.4.0 + resolution: "is-ssh@npm:1.4.0" + dependencies: + protocols: ^2.0.1 + checksum: 75eaa17b538bee24b661fbeb0f140226ac77e904a6039f787bea418431e2162f1f9c4c4ccad3bd169e036cd701cc631406e8c505d9fa7e20164e74b47f86f40f + languageName: node + linkType: hard + +"is-stream@npm:^1.0.0, is-stream@npm:^1.1.0": version: 1.1.0 resolution: "is-stream@npm:1.1.0" checksum: 063c6bec9d5647aa6d42108d4c59723d2bd4ae42135a2d4db6eadbd49b7ea05b750fd69d279e5c7c45cf9da753ad2c00d8978be354d65aa9f6bb434969c6a2ae @@ -26493,6 +26841,13 @@ __metadata: languageName: node linkType: hard +"iterate-object@npm:^1.1.0": + version: 1.3.4 + resolution: "iterate-object@npm:1.3.4" + checksum: b63496c489177babccb4b487322279ea4377e08d02b93902c3ffba3032a788f014a74e03c615da2a24807fa3fca872f69c9570f7801c8e88181df7a49298904b + languageName: node + linkType: hard + "iterate-value@npm:^1.0.2": version: 1.0.2 resolution: "iterate-value@npm:1.0.2" @@ -28822,6 +29177,15 @@ __metadata: languageName: node linkType: hard +"jsesc@npm:^3.0.2": + version: 3.0.2 + resolution: "jsesc@npm:3.0.2" + bin: + jsesc: bin/jsesc + checksum: a36d3ca40574a974d9c2063bf68c2b6141c20da8f2a36bd3279fc802563f35f0527a6c828801295bdfb2803952cf2cf387786c2c90ed564f88d5782475abfe3c + languageName: node + linkType: hard + "jsesc@npm:~0.5.0": version: 0.5.0 resolution: "jsesc@npm:0.5.0" @@ -29985,6 +30349,15 @@ __metadata: languageName: node linkType: hard +"limit-it@npm:^3.0.0": + version: 3.2.10 + resolution: "limit-it@npm:3.2.10" + dependencies: + typpy: ^2.0.0 + checksum: 3a809aad23f191d2abd469d835829391c8e85074de109a25af44c2d939c41fc92643ab8190f362191bd9157119f32574cebbc7eef6e99695f354a01f7b2a7b76 + languageName: node + linkType: hard + "lines-and-columns@npm:^1.1.6": version: 1.2.4 resolution: "lines-and-columns@npm:1.2.4" @@ -30397,6 +30770,13 @@ __metadata: languageName: node linkType: hard +"lowercase-keys@npm:^1.0.0": + version: 1.0.1 + resolution: "lowercase-keys@npm:1.0.1" + checksum: 4d045026595936e09953e3867722e309415ff2c80d7701d067546d75ef698dac218a4f53c6d1d0e7368b47e45fd7529df47e6cb56fbb90523ba599f898b3d147 + languageName: node + linkType: hard + "lru-cache@npm:10.2.2, lru-cache@npm:^10.2.2": version: 10.2.2 resolution: "lru-cache@npm:10.2.2" @@ -32435,6 +32815,13 @@ __metadata: languageName: node linkType: hard +"node-releases@npm:^2.0.18": + version: 2.0.18 + resolution: "node-releases@npm:2.0.18" + checksum: ef55a3d853e1269a6d6279b7692cd6ff3e40bc74947945101138745bfdc9a5edabfe72cb19a31a8e45752e1910c4c65c77d931866af6357f242b172b7283f5b3 + languageName: node + linkType: hard + "node-releases@npm:^2.0.8": version: 2.0.10 resolution: "node-releases@npm:2.0.10" @@ -32442,6 +32829,13 @@ __metadata: languageName: node linkType: hard +"node-status-codes@npm:^1.0.0": + version: 1.0.0 + resolution: "node-status-codes@npm:1.0.0" + checksum: 10fe52de31cc94536aa49a2a8a28e39a880d02832ac268e7edd2b082292232abcaa8e44fe4a318d072a08ce114851fab269ab8d7f9527bd5609aebaf2bb6df17 + languageName: node + linkType: hard + "non-layered-tidy-tree-layout@npm:^2.0.2": version: 2.0.2 resolution: "non-layered-tidy-tree-layout@npm:2.0.2" @@ -32449,6 +32843,13 @@ __metadata: languageName: node linkType: hard +"noop6@npm:^1.0.1": + version: 1.0.9 + resolution: "noop6@npm:1.0.9" + checksum: cc46d03eb22c5a990b3ce5e3ff71a82628efadcdeefdae566639657d2a947033ea919d1f4bfc7bfbcf674d43c21b3e580c2f07c8043e7ef487ac527237e2532f + languageName: node + linkType: hard + "nopt@npm:^5.0.0": version: 5.0.0 resolution: "nopt@npm:5.0.0" @@ -32482,7 +32883,7 @@ __metadata: languageName: node linkType: hard -"normalize-package-data@npm:^2.3.2, normalize-package-data@npm:^2.5.0": +"normalize-package-data@npm:^2.3.2, normalize-package-data@npm:^2.3.5, normalize-package-data@npm:^2.5.0": version: 2.5.0 resolution: "normalize-package-data@npm:2.5.0" dependencies: @@ -32946,6 +33347,16 @@ __metadata: languageName: node linkType: hard +"oargv@npm:^3.4.1": + version: 3.4.10 + resolution: "oargv@npm:3.4.10" + dependencies: + iterate-object: ^1.1.0 + ul: ^5.0.0 + checksum: f713a1995da354236ec87e8f4dcc21326fe3289129a2c3731b868432b845043ffc4a85ce390c734e25c99e2a8650b745b25282c5b4dffc61496d7f7432dd924d + languageName: node + linkType: hard + "oauth-sign@npm:~0.9.0": version: 0.9.0 resolution: "oauth-sign@npm:0.9.0" @@ -32953,6 +33364,15 @@ __metadata: languageName: node linkType: hard +"obj-def@npm:^1.0.0": + version: 1.0.9 + resolution: "obj-def@npm:1.0.9" + dependencies: + deffy: ^2.2.2 + checksum: 93e28098d186a5609968bf0aed4d0816f0e5baf0354093a6f79687d9e419bd26c38ff6a5674b009ed22987634cb915936cf077f5180a5dbd9e824c80a74c0064 + languageName: node + linkType: hard + "object-assign@npm:^4, object-assign@npm:^4.0.1, object-assign@npm:^4.1.0, object-assign@npm:^4.1.1": version: 4.1.1 resolution: "object-assign@npm:4.1.1" @@ -33205,6 +33625,16 @@ __metadata: languageName: node linkType: hard +"one-by-one@npm:^3.0.0, one-by-one@npm:^3.1.0": + version: 3.2.8 + resolution: "one-by-one@npm:3.2.8" + dependencies: + obj-def: ^1.0.0 + sliced: ^1.0.1 + checksum: 0af8cef27306172a67423001a1b8b242115ab733d18890275c9cd4adc8ec3414f0b453395915ca8eaddffbbbf71a68e74d16eded362dbd8fac22365af81277e1 + languageName: node + linkType: hard + "onetime@npm:^2.0.0": version: 2.0.1 resolution: "onetime@npm:2.0.1" @@ -33430,7 +33860,7 @@ __metadata: languageName: node linkType: hard -"os-tmpdir@npm:^1.0.1, os-tmpdir@npm:~1.0.2": +"os-tmpdir@npm:^1.0.1, os-tmpdir@npm:~1.0.1, os-tmpdir@npm:~1.0.2": version: 1.0.2 resolution: "os-tmpdir@npm:1.0.2" checksum: 5666560f7b9f10182548bf7013883265be33620b1c1b4a4d405c25be2636f970c5488ff3e6c48de75b55d02bde037249fe5dbfbb4c0fb7714953d56aed062e6d @@ -33681,6 +34111,38 @@ __metadata: languageName: node linkType: hard +"package-json-path@npm:^1.0.0": + version: 1.0.9 + resolution: "package-json-path@npm:1.0.9" + dependencies: + abs: ^1.2.1 + checksum: 4528ba905217628e5abf8bf843e7a5c6a9bb368068105ecca8c9d7d925922b5b9f982295710b0df63e1a51705b8077df449252a9fc17765e3b7e0358ae476860 + languageName: node + linkType: hard + +"package-json@npm:^2.3.1": + version: 2.4.0 + resolution: "package-json@npm:2.4.0" + dependencies: + got: ^5.0.0 + registry-auth-token: ^3.0.1 + registry-url: ^3.0.3 + semver: ^5.1.0 + checksum: 0120e4e8222e796f0b7c3e23c25d02a6cbb4818f4488f97e9dcd619383d56c8a7e7d9a0e46dee6a28b56ecabdbfa07ffca0049bff268ebf0ded0e7f1d906e1a0 + languageName: node + linkType: hard + +"package.json@npm:^2.0.1": + version: 2.0.1 + resolution: "package.json@npm:2.0.1" + dependencies: + git-package-json: ^1.4.0 + git-source: ^1.1.0 + package-json: ^2.3.1 + checksum: 579c4c3d6bdeb67e6e501bbaca82ee0a8c9590bf23c6da9eec8f7c2a4dcb3dc0b5dce9ce90ccfcde4c235c8269e1cebd84693d5123d16e5c1a0a22152579f6b6 + languageName: node + linkType: hard + "pacote@npm:^13.0.3": version: 13.1.1 resolution: "pacote@npm:13.1.1" @@ -33857,6 +34319,15 @@ __metadata: languageName: node linkType: hard +"parse-json@npm:^2.1.0": + version: 2.2.0 + resolution: "parse-json@npm:2.2.0" + dependencies: + error-ex: ^1.2.0 + checksum: dda78a63e57a47b713a038630868538f718a7ca0cd172a36887b0392ccf544ed0374902eb28f8bf3409e8b71d62b79d17062f8543afccf2745f9b0b2d2bb80ca + languageName: node + linkType: hard + "parse-json@npm:^4.0.0": version: 4.0.0 resolution: "parse-json@npm:4.0.0" @@ -33886,6 +34357,16 @@ __metadata: languageName: node linkType: hard +"parse-url@npm:^1.0.0": + version: 1.3.11 + resolution: "parse-url@npm:1.3.11" + dependencies: + is-ssh: ^1.3.0 + protocols: ^1.4.0 + checksum: 33e36566ed248cc8289a35c6c094f98cf68ee42735bd971f9696db4e75d4d85889e4cefecb4fbfab1da4990854e0a5b548434726aede7b9c72714d9dc45736c7 + languageName: node + linkType: hard + "parse5-htmlparser2-tree-adapter@npm:^6.0.0": version: 6.0.1 resolution: "parse5-htmlparser2-tree-adapter@npm:6.0.1" @@ -34154,6 +34635,13 @@ __metadata: languageName: node linkType: hard +"picocolors@npm:^1.1.0": + version: 1.1.1 + resolution: "picocolors@npm:1.1.1" + checksum: e1cf46bf84886c79055fdfa9dcb3e4711ad259949e3565154b004b260cd356c5d54b31a1437ce9782624bf766272fe6b0154f5f0c744fb7af5d454d2b60db045 + languageName: node + linkType: hard + "picomatch@npm:^2.0.4, picomatch@npm:^2.2.1, picomatch@npm:^2.2.2, picomatch@npm:^2.2.3, picomatch@npm:^2.3.0": version: 2.3.1 resolution: "picomatch@npm:2.3.1" @@ -35137,7 +35625,7 @@ __metadata: languageName: node linkType: hard -"prepend-http@npm:^1.0.0": +"prepend-http@npm:^1.0.0, prepend-http@npm:^1.0.1": version: 1.0.4 resolution: "prepend-http@npm:1.0.4" checksum: 01e7baf4ad38af02257b99098543469332fc42ae50df33d97a124bf8172295907352fa6138c9b1610c10c6dd0847ca736e53fda736387cc5cf8fcffe96b47f29 @@ -35447,6 +35935,20 @@ __metadata: languageName: node linkType: hard +"protocols@npm:^1.4.0": + version: 1.4.8 + resolution: "protocols@npm:1.4.8" + checksum: 2d555c013df0b05402970f67f7207c9955a92b1d13ffa503c814b5fe2f6dde7ac6a03320e0975c1f5832b0113327865e0b3b28bfcad023c25ddb54b53fab8684 + languageName: node + linkType: hard + +"protocols@npm:^2.0.1": + version: 2.0.1 + resolution: "protocols@npm:2.0.1" + checksum: 4a9bef6aa0449a0245ded319ac3cbfd032c3e76ebb562777037a3a832c99253d0e8bc2847f7be350236df620a11f7d4fe683ea7f59a2cc14c69f746b6259eda4 + languageName: node + linkType: hard + "proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" @@ -35904,6 +36406,25 @@ __metadata: languageName: node linkType: hard +"r-json@npm:^1.2.1": + version: 1.3.0 + resolution: "r-json@npm:1.3.0" + dependencies: + w-json: 1.3.10 + checksum: 9a2aa9b92a2f4b7932c7eb45175d9c7ff078e322eecaf1ca2c9cdda346ea68e73062004c1b3631a9127e84eedf982fc816110f0c7a1d07c6b2b3344f6d621791 + languageName: node + linkType: hard + +"r-package-json@npm:^1.0.0": + version: 1.0.9 + resolution: "r-package-json@npm:1.0.9" + dependencies: + package-json-path: ^1.0.0 + r-json: ^1.2.1 + checksum: e94b2b02e75dee37d42226656a66cacf69332324acf67743ddf4aa8300acc06bc35861c15379a4061232ca40d40d76f5a1beb82fe18b9e5d6225e2b722a3a138 + languageName: node + linkType: hard + "ramda@npm:0.29.0": version: 0.29.0 resolution: "ramda@npm:0.29.0" @@ -35998,7 +36519,7 @@ __metadata: languageName: node linkType: hard -"rc@npm:^1.2.7, rc@npm:^1.2.8": +"rc@npm:^1.0.1, rc@npm:^1.1.6, rc@npm:^1.2.7, rc@npm:^1.2.8": version: 1.2.8 resolution: "rc@npm:1.2.8" dependencies: @@ -36116,6 +36637,16 @@ __metadata: languageName: node linkType: hard +"read-all-stream@npm:^3.0.0": + version: 3.1.0 + resolution: "read-all-stream@npm:3.1.0" + dependencies: + pinkie-promise: ^2.0.0 + readable-stream: ^2.0.0 + checksum: ff7bf7c5484dcb6e857d9eccc388d2d32674e538c88cad61703e3d8ae9386a7fa74e8b30b6eb82a3979ea60413e933b90238e99a59f782d7fabebb7833a5b9c4 + languageName: node + linkType: hard + "read-cmd-shim@npm:^3.0.0": version: 3.0.1 resolution: "read-cmd-shim@npm:3.0.1" @@ -36236,6 +36767,21 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^2.0.5": + version: 2.3.8 + resolution: "readable-stream@npm:2.3.8" + dependencies: + core-util-is: ~1.0.0 + inherits: ~2.0.3 + isarray: ~1.0.0 + process-nextick-args: ~2.0.0 + safe-buffer: ~5.1.1 + string_decoder: ~1.1.1 + util-deprecate: ~1.0.1 + checksum: 65645467038704f0c8aaf026a72fbb588a9e2ef7a75cd57a01702ee9db1c4a1e4b03aaad36861a6a0926546a74d174149c8c207527963e0c2d3eee2f37678a42 + languageName: node + linkType: hard + "readdir-scoped-modules@npm:^1.1.0": version: 1.1.0 resolution: "readdir-scoped-modules@npm:1.1.0" @@ -36450,6 +36996,15 @@ __metadata: languageName: node linkType: hard +"regexp-tree@npm:^0.1.27": + version: 0.1.27 + resolution: "regexp-tree@npm:0.1.27" + bin: + regexp-tree: bin/regexp-tree + checksum: 129aebb34dae22d6694ab2ac328be3f99105143737528ab072ef624d599afecbcfae1f5c96a166fa9e5f64fa1ecf30b411c4691e7924c3e11bbaf1712c260c54 + languageName: node + linkType: hard + "regexp.prototype.flags@npm:^1.2.0, regexp.prototype.flags@npm:^1.3.1": version: 1.4.1 resolution: "regexp.prototype.flags@npm:1.4.1" @@ -36517,6 +37072,16 @@ __metadata: languageName: node linkType: hard +"registry-auth-token@npm:^3.0.1": + version: 3.4.0 + resolution: "registry-auth-token@npm:3.4.0" + dependencies: + rc: ^1.1.6 + safe-buffer: ^5.0.1 + checksum: a15780726bae327a8fff4048cb6a5de03d58bc19ea9e2411322e32e4ebb59962efb669d270bdde384ed68ed7b948f5feb11469e3d0c7e50a33cc8866710f0bc2 + languageName: node + linkType: hard + "registry-auth-token@npm:^4.0.0": version: 4.2.1 resolution: "registry-auth-token@npm:4.2.1" @@ -36526,6 +37091,15 @@ __metadata: languageName: node linkType: hard +"registry-url@npm:^3.0.3": + version: 3.1.0 + resolution: "registry-url@npm:3.1.0" + dependencies: + rc: ^1.0.1 + checksum: 6d223da41b04e1824f5faa63905c6f2e43b216589d72794111573f017352b790aef42cd1f826463062f89d804abb2027e3d9665d2a9a0426a11eedd04d470af3 + languageName: node + linkType: hard + "regjsgen@npm:^0.6.0": version: 0.6.0 resolution: "regjsgen@npm:0.6.0" @@ -36533,6 +37107,17 @@ __metadata: languageName: node linkType: hard +"regjsparser@npm:^0.10.0": + version: 0.10.0 + resolution: "regjsparser@npm:0.10.0" + dependencies: + jsesc: ~0.5.0 + bin: + regjsparser: bin/parser + checksum: 17550661f43ba792f8365fb95b3dbdb64e25f14e31ef7c2c11876c240a60e87b7bfc28c98589f4e76b7cf49307e45fb24d030f57d68dd0cc41c56b4d378e9254 + languageName: node + linkType: hard + "regjsparser@npm:^0.8.2": version: 0.8.4 resolution: "regjsparser@npm:0.8.4" @@ -37684,6 +38269,15 @@ resolve@1.1.7: languageName: node linkType: hard +"semver@npm:^5.1.0": + version: 5.7.2 + resolution: "semver@npm:5.7.2" + bin: + semver: bin/semver + checksum: fb4ab5e0dd1c22ce0c937ea390b4a822147a9c53dbd2a9a0132f12fe382902beef4fbf12cf51bb955248d8d15874ce8cd89532569756384f994309825f10b686 + languageName: node + linkType: hard + "semver@npm:^6.0.0, semver@npm:^6.1.0, semver@npm:^6.1.1, semver@npm:^6.1.2, semver@npm:^6.2.0, semver@npm:^6.3.0": version: 6.3.0 resolution: "semver@npm:6.3.0" @@ -38212,6 +38806,13 @@ resolve@1.1.7: languageName: node linkType: hard +"sliced@npm:^1.0.1": + version: 1.0.1 + resolution: "sliced@npm:1.0.1" + checksum: 84528d23279985ead75809eeec5d601b0fb6bc28348c6627f4feb40747533a1e36a75e8bc60f9079528079b21c434890b397e8fc5c24a649165cc0bbe90b4d70 + languageName: node + linkType: hard + "smart-buffer@npm:^4.2.0": version: 4.2.0 resolution: "smart-buffer@npm:4.2.0" @@ -39855,6 +40456,13 @@ resolve@1.1.7: languageName: node linkType: hard +"timed-out@npm:^3.0.0": + version: 3.1.3 + resolution: "timed-out@npm:3.1.3" + checksum: 7952bcc926fd43f3206f7cb9f27dc54912a48c08058da0208d33dabef35b6759d9f1d2d14976c529d2209bd92a109bb057d377211fd2d2d962f4ca0a8d3f15f0 + languageName: node + linkType: hard + "timers-browserify@npm:^2.0.4": version: 2.0.12 resolution: "timers-browserify@npm:2.0.12" @@ -39901,6 +40509,15 @@ resolve@1.1.7: languageName: node linkType: hard +"tmp@npm:0.0.28": + version: 0.0.28 + resolution: "tmp@npm:0.0.28" + dependencies: + os-tmpdir: ~1.0.1 + checksum: 8167d2471b650f88fde1515bbf6c62d2adef07972d06531569f789863a2436c32027b6d5fbe3102ed2c430b86043dffd337db12494f1f982534862d9487e4eff + languageName: node + linkType: hard + "tmp@npm:^0.0.33": version: 0.0.33 resolution: "tmp@npm:0.0.33" @@ -40817,6 +41434,15 @@ typescript@~4.4.3: languageName: node linkType: hard +"typpy@npm:^2.0.0, typpy@npm:^2.1.0, typpy@npm:^2.2.0, typpy@npm:^2.3.4": + version: 2.3.13 + resolution: "typpy@npm:2.3.13" + dependencies: + function.name: ^1.0.3 + checksum: e415782245876bd2bd069e227feb93fb0e4d93ccc4c1f10c0bd2a2ba223a92dfca18a4ed95b8c87eb226337909f4fa1509c6e962d7917f6d648e7d24b497a13e + languageName: node + linkType: hard + "uglify-js@npm:3.4.x": version: 3.4.10 resolution: "uglify-js@npm:3.4.10" @@ -40838,6 +41464,16 @@ typescript@~4.4.3: languageName: node linkType: hard +"ul@npm:^5.0.0": + version: 5.2.15 + resolution: "ul@npm:5.2.15" + dependencies: + deffy: ^2.2.2 + typpy: ^2.3.4 + checksum: a47735d307da2e9b25568977d8b13d5ab2845b2f7a14ad75b22022945f3879e0ecfef8b6134cb661fc2e46a2e86e3f69c44659816c422d0be50181aeaf44413d + languageName: node + linkType: hard + "unbox-primitive@npm:^1.0.1": version: 1.0.1 resolution: "unbox-primitive@npm:1.0.1" @@ -41124,6 +41760,13 @@ typescript@~4.4.3: languageName: node linkType: hard +"unzip-response@npm:^1.0.2": + version: 1.0.2 + resolution: "unzip-response@npm:1.0.2" + checksum: 09efe5d1d23a40534f5f67f268c6f4d2533cba4f2d40ae3233e50f9d7d3ee1db5503c52014f3265174517fff9fdebedd9ac6ec0c57b23a4933791d754b722187 + languageName: node + linkType: hard + "upath@npm:^1.1.1": version: 1.2.0 resolution: "upath@npm:1.2.0" @@ -41173,6 +41816,20 @@ typescript@~4.4.3: languageName: node linkType: hard +"update-browserslist-db@npm:^1.1.0": + version: 1.1.1 + resolution: "update-browserslist-db@npm:1.1.1" + dependencies: + escalade: ^3.2.0 + picocolors: ^1.1.0 + peerDependencies: + browserslist: ">= 4.21.0" + bin: + update-browserslist-db: cli.js + checksum: 2ea11bd2562122162c3e438d83a1f9125238c0844b6d16d366e3276d0c0acac6036822dc7df65fc5a89c699cdf9f174acf439c39bedf3f9a2f3983976e4b4c3e + languageName: node + linkType: hard + "upper-case@npm:^1.1.1": version: 1.1.3 resolution: "upper-case@npm:1.1.3" @@ -41220,6 +41877,15 @@ typescript@~4.4.3: languageName: node linkType: hard +"url-parse-lax@npm:^1.0.0": + version: 1.0.0 + resolution: "url-parse-lax@npm:1.0.0" + dependencies: + prepend-http: ^1.0.1 + checksum: 03316acff753845329652258c16d1688765ee34f7d242a94dadf9ff6e43ea567ec062cec7aa27c37f76f2c57f95e0660695afff32fb97b527591c7340a3090fa + languageName: node + linkType: hard + "url-parse@npm:^1.5.3": version: 1.5.10 resolution: "url-parse@npm:1.5.10" @@ -41780,6 +42446,13 @@ typescript@~4.4.3: languageName: node linkType: hard +"w-json@npm:1.3.10": + version: 1.3.10 + resolution: "w-json@npm:1.3.10" + checksum: 8535a207e579e616797efc4d5140acc7c0aefd11f0c9f846e6739816a2db8637d235492d86fc5c47bb2dba5821413d72b2d62df9184ee9d6e22e67b3f90d205b + languageName: node + linkType: hard + "w3c-hr-time@npm:^1.0.1, w3c-hr-time@npm:^1.0.2": version: 1.0.2 resolution: "w3c-hr-time@npm:1.0.2" @@ -43005,6 +43678,21 @@ typescript@~4.4.3: languageName: node linkType: hard +"yargs@npm:^17.7.2": + version: 17.7.2 + resolution: "yargs@npm:17.7.2" + dependencies: + cliui: ^8.0.1 + escalade: ^3.1.1 + get-caller-file: ^2.0.5 + require-directory: ^2.1.1 + string-width: ^4.2.3 + y18n: ^5.0.5 + yargs-parser: ^21.1.1 + checksum: 73b572e863aa4a8cbef323dd911d79d193b772defd5a51aab0aca2d446655216f5002c42c5306033968193bdbf892a7a4c110b0d77954a7fdf563e653967b56a + languageName: node + linkType: hard + "yarn-changed-workspaces@npm:^2.0.9": version: 2.0.10 resolution: "yarn-changed-workspaces@npm:2.0.10" From 489bfdf7f1e1598a559d2e6fe135973ee9552f0d Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 21 Oct 2024 18:05:53 -0400 Subject: [PATCH 09/15] feat: Refactor fulltext search to use @appland/search --- packages/cli/package.json | 1 + packages/cli/src/fulltext/FileIndex.ts | 300 ------------------ packages/cli/src/fulltext/SourceIndex.ts | 151 --------- .../cli/src/fulltext/listGitProjectFIles.ts | 59 ---- packages/cli/src/fulltext/listProjectFiles.ts | 41 --- packages/cli/src/fulltext/querySymbols.ts | 62 ---- packages/cli/src/fulltext/withIndex.ts | 39 --- .../src/rpc/explain/SearchContextCollector.ts | 109 ++++--- .../cli/src/rpc/explain/SourceCollector.ts | 28 -- .../cli/src/rpc/explain/collect-snippets.ts | 22 ++ packages/cli/src/rpc/explain/index-files.ts | 49 +++ .../cli/src/rpc/explain/index-snippets.ts | 21 ++ .../cli/tests/unit/fulltext/FileIndex.spec.ts | 186 ----------- .../tests/unit/fulltext/SourceIndex.spec.ts | 52 --- .../unit/fulltext/listGitProjectFiles.spec.ts | 50 --- .../unit/fulltext/listProjectFiles.spec.ts | 60 ---- .../tests/unit/fulltext/queryKeywords.spec.ts | 83 ----- .../tests/unit/fulltext/querySymbols.spec.ts | 188 ----------- .../explain/SearchContextCollector.spec.ts | 130 -------- .../unit/rpc/explain/collectContext.spec.ts | 124 ++++++++ 20 files changed, 270 insertions(+), 1485 deletions(-) delete mode 100644 packages/cli/src/fulltext/FileIndex.ts delete mode 100644 packages/cli/src/fulltext/SourceIndex.ts delete mode 100644 packages/cli/src/fulltext/listGitProjectFIles.ts delete mode 100644 packages/cli/src/fulltext/listProjectFiles.ts delete mode 100644 packages/cli/src/fulltext/querySymbols.ts delete mode 100644 packages/cli/src/fulltext/withIndex.ts delete mode 100644 packages/cli/src/rpc/explain/SourceCollector.ts create mode 100644 packages/cli/src/rpc/explain/collect-snippets.ts create mode 100644 packages/cli/src/rpc/explain/index-files.ts create mode 100644 packages/cli/src/rpc/explain/index-snippets.ts delete mode 100644 packages/cli/tests/unit/fulltext/FileIndex.spec.ts delete mode 100644 packages/cli/tests/unit/fulltext/SourceIndex.spec.ts delete mode 100644 packages/cli/tests/unit/fulltext/listGitProjectFiles.spec.ts delete mode 100644 packages/cli/tests/unit/fulltext/listProjectFiles.spec.ts delete mode 100644 packages/cli/tests/unit/fulltext/queryKeywords.spec.ts delete mode 100644 packages/cli/tests/unit/fulltext/querySymbols.spec.ts delete mode 100644 packages/cli/tests/unit/rpc/explain/SearchContextCollector.spec.ts create mode 100644 packages/cli/tests/unit/rpc/explain/collectContext.spec.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 559c48e29e..0e85f6074e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -94,6 +94,7 @@ "@appland/openapi": "workspace:^1.8.0", "@appland/rpc": "workspace:^1.13.0", "@appland/scanner": "workspace:^1.86.0", + "@appland/search": "workspace:^1.0.0", "@appland/sequence-diagram": "workspace:^1.12.0", "@octokit/rest": "^20.0.1", "@sidvind/better-ajv-errors": "^0.9.1", diff --git a/packages/cli/src/fulltext/FileIndex.ts b/packages/cli/src/fulltext/FileIndex.ts deleted file mode 100644 index 4dd6e54030..0000000000 --- a/packages/cli/src/fulltext/FileIndex.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { stat } from 'node:fs/promises'; -import path, { join } from 'node:path'; - -import sqlite3 from 'better-sqlite3'; -import assert from 'assert'; -import makeDebug from 'debug'; -import { existsSync } from 'fs'; - -import listProjectFiles from './listProjectFiles'; -import queryKeywords from './queryKeywords'; -import { Git, GitState } from '../telemetry'; -import listGitProjectFiles from './listGitProjectFIles'; -import querySymbols from './querySymbols'; -import { fileNameMatchesFilterPatterns } from './fileNameMatchesFilterPatterns'; - -makeDebug.formatters.a = (v) => { - return v.join(' '); -}; - -const debug = makeDebug('appmap:file-index'); - -// enable to see all indexing terms for each file -const debugTerms = makeDebug('appmap:file-index:terms'); - -export type FileIndexMatch = { - directory: string; - fileName: string; - score: number; -}; - -type ParsingOptions = { - allowGenericParsing?: boolean; - allowSymbols?: boolean; -}; - -export class FileIndex { - constructor(public database: sqlite3.Database) { - this.database.exec( - `CREATE VIRTUAL TABLE files USING fts5(directory UNINDEXED, file_name UNINDEXED, terms, tokenize = 'porter unicode61')` - ); - this.database.pragma('journal_mode = OFF'); - this.database.pragma('synchronous = OFF'); - this.#insert = this.database.prepare( - 'INSERT INTO files (directory, file_name, terms) VALUES (?, ?, ?)' - ); - } - - close() { - this.database.close(); - } - - search(keywords: string[], limit = 10): FileIndexMatch[] { - const query = `SELECT directory, file_name, (rank * -1) score FROM files WHERE files MATCH ? ORDER BY rank LIMIT ?`; - - const searchExpr = queryKeywords(keywords).join(' OR '); - const rows = this.database.prepare(query).all(searchExpr, limit); - if (debug.enabled) - for (const row of rows) { - debug('Found row %s', (row as { file_name: string }).file_name); - } - return rows.map((row: any) => ({ - directory: row.directory, - fileName: row.file_name, - score: row.score, - })); - } - - async indexDirectories( - directories: string[], - excludePatterns: RegExp[] | undefined, - includePatterns: RegExp[] | undefined, - batchSize = 100 - ) { - for (const directory of directories) { - try { - const startTime = new Date().getTime(); - const gitState = await Git.state(directory); - const fileNames = - gitState === GitState.Ok - ? await listGitProjectFiles(directory) - : await listProjectFiles(directory); - - const filteredFileNames = await filterFiles( - directory, - fileNames, - excludePatterns, - includePatterns - ); - - const options = { - allowGenericParsing: fileNames.length < 15_000, - allowSymbols: fileNames.length < 15_000, - }; - - if (options.allowSymbols) { - debug('Symbol parsing is enabled.'); - debug( - 'Generic symbol parsing is %s', - options.allowGenericParsing ? 'enabled.' : 'disabled.' - ); - } else { - debug('Symbol parsing is disabled.'); - } - - for (let i = 0; i < filteredFileNames.length; i += batchSize) { - this.indexDirectory(directory, filteredFileNames, options, i, batchSize); - - // yield to the event loop after each chunk - await new Promise((resolve) => setTimeout(resolve, 0)); - } - - const endTime = new Date().getTime(); - console.log( - `Indexed ${fileNames.length} files in ${directory} in ${endTime - startTime}ms` - ); - } catch (error) { - console.error(`Error processing directory ${directory}:`, error); - } - } - } - - private indexDirectory = this.database.transaction( - ( - directory: string, - fileNames: string[], - options?: ParsingOptions, - offset?: number, - limit?: number - ) => { - const startIndex = offset ?? 0; - const endIndex = limit ? Math.min(startIndex + limit, fileNames.length) : fileNames.length; - for (let i = startIndex; i < endIndex; i++) { - const fileName = fileNames[i]; - this.indexFile(directory, fileName, options); - } - } - ); - - #insert: sqlite3.Statement; - - indexFile(directory: string, filePath: string, options: ParsingOptions = {}) { - const { allowGenericParsing = true, allowSymbols = true } = options; - const fileNameTokens = filePath.split(path.sep); - - try { - let terms = queryKeywords(fileNameTokens); - - if (allowSymbols) { - const symbols = querySymbols(path.join(directory, filePath), allowGenericParsing); - terms = terms.concat(queryKeywords(symbols).sort()); - } - - debug('Indexing file path %s with %d terms', filePath, terms.length); - debugTerms('Terms for path %s: %a', filePath, terms); - - this.#insert.run(directory, filePath, terms.join(' ')); - } catch (error) { - console.warn(`Error indexing document ${filePath}`); - console.warn(error); - } - } -} - -export function restoreFileIndex(indexFileName: string): FileIndex { - assert(existsSync(indexFileName), `Index file ${indexFileName} does not exist`); - const database = new sqlite3(indexFileName); - return new FileIndex(database); -} - -export async function buildFileIndex( - directories: string[], - indexFileName: string, - excludePatterns?: RegExp[], - includePatterns?: RegExp[] -): Promise { - assert(!existsSync(indexFileName), `Index file ${indexFileName} already exists`); - const database = new sqlite3(indexFileName); - const fileIndex = new FileIndex(database); - await fileIndex.indexDirectories(directories, excludePatterns, includePatterns); - console.log(`Wrote file index to ${indexFileName}`); - return fileIndex; -} - -const BINARY_FILE_EXTENSIONS: string[] = [ - '7z', - 'aac', - 'avi', - 'bmp', - 'bz2', - 'class', - 'dll', - 'doc', - 'docx', - 'dylib', - 'ear', - 'exe', - 'eot', - 'flac', - 'flv', - 'gif', - 'gz', - 'ico', - 'jar', - 'jpeg', - 'jpg', - 'js.map', - 'min.js', - 'min.css', - 'mkv', - 'mo', - 'mov', - 'mp3', - 'mp4', - 'mpg', - 'odt', - 'odp', - 'ods', - 'ogg', - 'otf', - 'pdf', - 'po', - 'png', - 'ppt', - 'pptx', - 'pyc', - 'rar', - 'rtf', - 'so', - 'svg', - 'tar', - 'tiff', - 'ttf', - 'wav', - 'webm', - 'webp', - 'woff', - 'woff2', - 'wmv', - 'xls', - 'xlsx', - 'xz', - 'yarn.lock', - 'zip', -].map((ext) => '.' + ext); - -const DATA_FILE_EXTENSIONS: string[] = [ - 'csv', - 'dat', - 'log', - 'json', - 'tsv', - 'yaml', - 'yml', - 'xml', -].map((ext) => '.' + ext); - -const isBinaryFile = (fileName: string) => { - return BINARY_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext)); -}; - -const isDataFile = (fileName: string) => { - return DATA_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext)); -}; - -export async function filterFiles( - directory: string, - fileNames: string[], - excludePatterns?: RegExp[], - includePatterns?: RegExp[] -): Promise { - const result: string[] = []; - for (const fileName of fileNames) { - if (isBinaryFile(fileName)) continue; - - const includeFile = fileNameMatchesFilterPatterns(fileName, includePatterns, excludePatterns); - if (!includeFile) continue; - - let appendFile = false; - try { - const stats = await stat(join(directory, fileName)); - if (stats.isFile()) { - appendFile = true; - if (stats.size > 50_000) { - if (isDataFile(fileName)) { - debug('Skipping large data file %s with size %d', fileName, stats.size); - appendFile = false; - } else { - debug('WARNING Large file %s with size %d', fileName, stats.size); - } - } - } - } catch (error) { - console.warn(`Error checking file ${fileName}`); - console.warn(error); - } - - if (appendFile) result.push(fileName); - } - return result; -} diff --git a/packages/cli/src/fulltext/SourceIndex.ts b/packages/cli/src/fulltext/SourceIndex.ts deleted file mode 100644 index 94699e94a1..0000000000 --- a/packages/cli/src/fulltext/SourceIndex.ts +++ /dev/null @@ -1,151 +0,0 @@ -import makeDebug from 'debug'; -import { - RecursiveCharacterTextSplitter, - SupportedTextSplitterLanguage, -} from 'langchain/text_splitter'; -import type { Document } from 'langchain/document'; -import assert from 'assert'; -import sqlite3 from 'better-sqlite3'; -import { existsSync } from 'fs'; -import { FileIndexMatch } from './FileIndex'; -import { readFile } from 'fs/promises'; -import { join } from 'path'; -import queryKeywords from './queryKeywords'; - -const debug = makeDebug('appmap:source-index'); - -const TEXT_SPLITTER_LANGUAGE_EXTENSIONS: Record = { - cpp: ['cpp', 'h', 'hpp', 'c', 'cc', 'cxx', 'hxx'], - go: ['go'], - java: ['java', 'jsp', 'jspx'], - js: ['js', 'ts', 'mjs', 'jsx', 'tsx', 'vue', 'svelte'], - php: ['php'], - proto: ['proto'], - python: ['py'], - rst: ['rst'], - ruby: ['rb', 'haml', 'erb'], - rust: ['rs'], - scala: ['scala'], - swift: ['swift'], - markdown: ['md'], - latex: ['tex'], - html: ['html'], - sol: ['sol'], -}; - -export type SourceIndexDocument = { - directory: string; - fileName: string; -}; - -export type SourceIndexMatch = { - directory: string; - fileName: string; - from: number; - to: number; - content: string; - score: number; -}; - -export class SourceIndex { - constructor(public database: sqlite3.Database) { - this.database.exec( - `CREATE VIRTUAL TABLE code_snippets USING fts5(directory UNINDEXED, file_name UNINDEXED, from_line UNINDEXED, to_line UNINDEXED, snippet UNINDEXED, terms, tokenize = 'porter unicode61')` - ); - this.database.pragma('journal_mode = OFF'); - this.database.pragma('synchronous = OFF'); - this.#insert = this.database.prepare( - 'INSERT INTO code_snippets (directory, file_name, from_line, to_line, snippet, terms) VALUES (?, ?, ?, ?, ?, ?)' - ); - } - - close() { - this.database.close(); - } - - search(keywords: string[], limit = 10): SourceIndexMatch[] { - const query = `SELECT directory, file_name, from_line, to_line, snippet, (rank * -1) score FROM code_snippets WHERE code_snippets MATCH ? ORDER BY rank LIMIT ?`; - - const searchExpr = queryKeywords(keywords).join(' OR '); - debug(`[SourceIndex] Searching for ${searchExpr}`); - const rows = this.database.prepare(query).all(searchExpr, limit); - if (debug.enabled) - rows.forEach((row: any) => { - debug(`[SourceIndex] Found row ${row.file_name}`); - }); - return rows.map((row: any) => ({ - directory: row.directory, - fileName: row.file_name, - from: row.from_line, - to: row.to_line, - content: row.snippet, - score: row.score, - })); - } - - async indexFiles(files: SourceIndexDocument[]): Promise { - for (const file of files) { - await this.indexFile(file); - } - } - - #insert: sqlite3.Statement; - - async indexFile(file: SourceIndexDocument) { - const { directory, fileName } = file; - const extension = fileName.split('.').pop(); - - const language = Object.keys(TEXT_SPLITTER_LANGUAGE_EXTENSIONS).find((language) => - TEXT_SPLITTER_LANGUAGE_EXTENSIONS[language].includes(extension) - ) as SupportedTextSplitterLanguage | undefined; - let splitter: RecursiveCharacterTextSplitter; - if (language) { - splitter = RecursiveCharacterTextSplitter.fromLanguage(language); - } else { - debug(`No language found for file: ${fileName}`); - splitter = new RecursiveCharacterTextSplitter(); - } - const filePath = join(directory, fileName); - let fileContents: string; - try { - fileContents = await readFile(filePath, 'utf-8'); - } catch (error) { - console.warn(`Error reading file ${filePath}`); - console.warn(error); - return; - } - - const chunks = await splitter.createDocuments([fileContents]); - this.#indexChunks(chunks, directory, fileName); - } - - #indexChunks = this.database.transaction( - (chunks: Document[], directory: string, fileName: string) => { - for (const chunk of chunks) { - const { from, to } = chunk.metadata.loc.lines; - - try { - debug(`Indexing document ${fileName} from ${from} to ${to}`); - - const terms = queryKeywords([fileName, chunk.pageContent]).join(' '); - this.#insert.run(directory, fileName, from, to, chunk.pageContent, terms); - } catch (error) { - console.warn(`Error indexing document ${fileName} from ${from} to ${to}`); - console.warn(error); - } - } - } - ); -} - -export async function buildSourceIndex( - indexFileName: string, - files: FileIndexMatch[] -): Promise { - assert(!existsSync(indexFileName), `Index file ${indexFileName} already exists`); - const database = new sqlite3(indexFileName); - const sourceIndex = new SourceIndex(database); - await sourceIndex.indexFiles(files); - console.log(`Wrote file index to ${indexFileName}`); - return sourceIndex; -} diff --git a/packages/cli/src/fulltext/listGitProjectFIles.ts b/packages/cli/src/fulltext/listGitProjectFIles.ts deleted file mode 100644 index b7261058e1..0000000000 --- a/packages/cli/src/fulltext/listGitProjectFIles.ts +++ /dev/null @@ -1,59 +0,0 @@ -import makeDebug from 'debug'; - -import { verbose } from '../utils'; -import { exec as execCb } from 'node:child_process'; -import { promisify } from 'util'; - -const debug = makeDebug('appmap:listGitProjectFiles'); -const exec = promisify(execCb); - -// Run git ls-files and git status to get a list of all git-managed files. By doing it this way, -// we automatically apply any .gitignore rules. -export default async function listGitProjectFiles(directory: string): Promise { - const lsFiles = async (): Promise => { - try { - const { stdout } = await exec('git ls-files', { - cwd: directory, - maxBuffer: 1024 ** 2 * 20, // 20 MB - }); - - debug(stdout); - - return stdout.split('\n').filter(Boolean); - } catch (e) { - if (verbose()) { - console.error('`git ls-files` failed'); - console.error(e); - } - return []; - } - }; - - const statusFiles = async (): Promise => { - try { - const { stdout } = await exec('git status --porcelain', { - cwd: directory, - maxBuffer: 1024 ** 2 * 20, // 20 MB - }); - - debug(stdout); - - return stdout - .split('\n') - .map((line) => { - // git status --porcelain output starts with 3 characters: staged status, unstaged status, - // and a space. - return line.slice(3); - }) - .filter(Boolean); - } catch (e) { - if (verbose()) { - console.error('`git status --porcelain` failed'); - console.error(e); - } - return []; - } - }; - - return Array.from(new Set([...(await lsFiles()), ...(await statusFiles())])); -} diff --git a/packages/cli/src/fulltext/listProjectFiles.ts b/packages/cli/src/fulltext/listProjectFiles.ts deleted file mode 100644 index d4db31c5e2..0000000000 --- a/packages/cli/src/fulltext/listProjectFiles.ts +++ /dev/null @@ -1,41 +0,0 @@ -import assert from 'assert'; -import { readdir } from 'fs/promises'; -import { join, relative } from 'path'; - -const IGNORE_DIRECTORIES = ['node_modules', 'vendor', 'tmp', 'build', 'dist', 'target']; - -const DEFAULT_PROJECT_FILE_LIMIT = 1000; - -// Produce a modest-sized listing of files in the project. -// Ignore a standard list of binary file extensions and directories that tend to be full of -// non-source files. -export default async function listProjectFiles( - directory: string, - fileLimit = DEFAULT_PROJECT_FILE_LIMIT -): Promise { - const files = new Array(); - - const ignoreDirectory = (dir: string) => IGNORE_DIRECTORIES.includes(dir); - - // Perform a breadth-first traversal of a directory, collecting all non-binary files and - // applying the directory ignore list. - const processDir = async (dir: string) => { - const queue = [dir]; - while (queue.length > 0 && files.length < fileLimit) { - const currentDir = queue.shift(); - assert(currentDir, 'queue should not be empty'); - - const entries = await readdir(currentDir, { withFileTypes: true }); - for (const entry of entries) { - const path = join(currentDir, entry.name); - if (entry.isDirectory()) { - if (!ignoreDirectory(entry.name)) queue.push(path); - } else files.push(relative(dir, path)); - } - } - }; - - await processDir(directory); - - return files; -} diff --git a/packages/cli/src/fulltext/querySymbols.ts b/packages/cli/src/fulltext/querySymbols.ts deleted file mode 100644 index fca2963f48..0000000000 --- a/packages/cli/src/fulltext/querySymbols.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { readFileSync } from 'fs'; - -export const SymbolRegexes: Record = { - cs: /(((interface|class|enum|struct)\s+(?\w+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, - cpp: /(((struct|enum|union|class)\s+(?\w+)\s*?\{)|(}\s*?(?\w+)\s*?;)|((\s|^)(?!try|catch|if|while|do|for|switch)(?[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, - rs: /(struct|enum|union|trait|type|fn)\s+(?[\w\p{L}]+)/gu, - go: /((type\s+(?[\w\p{L}]+))|(func\s+?(\(.*?\)\s*?)?(?[\w\p{L}]+)\s*?\())/gu, - rb: /(((class|module)\s+(?\w+))|(def\s+?(?\w+)))/g, - py: /(class|def)\s+(?\w+)/g, - java: /(((class|@?interface|enum)\s+(?[\w$]+))|((\s|^)(?!try|catch|if|while|do|for|switch)(?[\w$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g, - ts: /(((class|interface|enum|type|function)\s+(?[#$\w\p{L}]+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?[#$\w\p{L}]+)\s*?\([^;)]*?\)[\w\s<>[\].:\n]*?\{)|((?[#$\w\p{L}]+)\s*?(=|:)\s*?\(.*?\)\s*?=>))/gu, - kt: /(((class|typealias)\s+(?[\w_]+))|(fun\s+?(<.+?>\s+)?(.*?\.)?(?\w+)))/g, - php: /(class|trait|function)\s+(?[\w_$]+)/g, -}; - -const genericSymbolRegex = - /(((^|\s)(?!using|try|catch|if|while|do|for|switch)(?[#$\w\p{L}~]+)\s*?\(([^;)])*?\)[\w\s<>[\].:\n]*?\{)|(^(?!.*?(?:#|\/\/|"|')).*?(interface|class|enum|struct|union|trait|type(alias|def)?|fu?nc?(tion)?|module|def)\s+?(?[#$\w\p{L}]+))|((?[#$\w\p{L}~]+)\s*?=\s*?[\w\s<>[\].:\n]*?\{))/gmu; - -// Define aliases for common file extensions -['js', 'jsx', 'ts', 'tsx', 'vue', 'svelte'].forEach((ext) => { - SymbolRegexes[ext] = SymbolRegexes.ts; -}); - -['c', 'cc', 'cxx', 'h', 'hpp', 'cpp', 'hxx'].forEach((ext) => { - SymbolRegexes[ext] = SymbolRegexes.cpp; -}); - -function getMatches(source: string, regex: RegExp): string[] { - const results: string[] = []; - const matches = source.matchAll(regex); - - for (const match of matches) { - const { groups } = match; - const symbol = groups?.symbol1 ?? groups?.symbol2 ?? groups?.symbol3; - - if (symbol) results.push(symbol); - } - - return results; -} - -export default function querySymbols(path: string, allowGeneric = true): string[] { - const fileExtension = path.split('.').pop()?.toLowerCase(); - - let regex = allowGeneric ? genericSymbolRegex : undefined; - if (fileExtension && fileExtension in SymbolRegexes) { - regex = SymbolRegexes[fileExtension]; - } - - if (regex) { - try { - // readFileSync performs 2x faster than its async counterpart in this case. - const content = readFileSync(path, 'utf-8'); - return getMatches(content, regex); - } catch (error) { - console.warn(`Error reading file ${path}`); - console.warn(error); - } - } - - return []; -} diff --git a/packages/cli/src/fulltext/withIndex.ts b/packages/cli/src/fulltext/withIndex.ts deleted file mode 100644 index e059efd08e..0000000000 --- a/packages/cli/src/fulltext/withIndex.ts +++ /dev/null @@ -1,39 +0,0 @@ -// This function is used for a "one-shot" search in which the index us built, used, and discarded. -// Replace this with a persistent index file that can be used across multiple searches, and is - -import { mkdtemp, rm } from 'fs/promises'; -import { tmpdir } from 'os'; -import { join } from 'path'; - -export interface Closeable { - close(): void; -} - -// synced with the file system as needed. -export default async function withIndex( - indexName: string, - builder: (indexFile: string) => Promise, - callback: (index: T) => S | Promise -): Promise { - const tmpDir = await mkdtemp(join(tmpdir(), `appmap-${indexName}-${new Date().getTime()}`)); - const indexFile = join(tmpDir, 'index.sqlite'); - - const close = (index: T) => { - try { - index.close(); - } catch (err) { - console.error(err); - } - }; - const cleanupDir = async () => - await rm(tmpDir, { recursive: true }).catch((err) => console.error(err)); - - let index: T | undefined; - try { - index = await builder(indexFile); - return await callback(index); - } finally { - if (index) close(index); - await cleanupDir(); - } -} diff --git a/packages/cli/src/rpc/explain/SearchContextCollector.ts b/packages/cli/src/rpc/explain/SearchContextCollector.ts index bfc481d67f..5d85182975 100644 --- a/packages/cli/src/rpc/explain/SearchContextCollector.ts +++ b/packages/cli/src/rpc/explain/SearchContextCollector.ts @@ -1,32 +1,30 @@ import { log } from 'console'; +import sqlite3 from 'better-sqlite3'; + import { ContextV2, applyContext } from '@appland/navie'; import { SearchRpc } from '@appland/rpc'; import AppMapIndex, { SearchResponse as AppMapSearchResponse, SearchOptions as AppMapSearchOptions, } from '../../fulltext/AppMapIndex'; -import { DEFAULT_MAX_DIAGRAMS, DEFAULT_MAX_FILES } from '../search/search'; -import { buildFileIndex } from '../../fulltext/FileIndex'; -import withIndex from '../../fulltext/withIndex'; +import { DEFAULT_MAX_DIAGRAMS } from '../search/search'; import EventCollector from './EventCollector'; -import SourceCollector from './SourceCollector'; +import indexFiles from './index-files'; +import indexSnippets from './index-snippets'; +import collectSnippets from './collect-snippets'; export default class SearchContextCollector { public excludePatterns: RegExp[] | undefined; public includePatterns: RegExp[] | undefined; public includeTypes: ContextV2.ContextItemType[] | undefined; - query: string; - constructor( private appmapDirectories: string[], private sourceDirectories: string[], private appmaps: string[] | undefined, private vectorTerms: string[], private charLimit: number - ) { - this.query = this.vectorTerms.join(' '); - } + ) {} async collectContext(): Promise<{ searchResponse: SearchRpc.SearchResponse; @@ -64,67 +62,66 @@ export default class SearchContextCollector { }; appmapSearchResponse = await AppMapIndex.search( this.appmapDirectories, - this.query, + this.vectorTerms.join(' '), searchOptions ); } - const fileSearchResponse = await withIndex( - 'files', - (indexFileName: string) => - buildFileIndex( - this.sourceDirectories, - indexFileName, - this.excludePatterns, - this.includePatterns - ), - (index) => index.search(this.vectorTerms, DEFAULT_MAX_FILES) - ); - - const eventsCollector = new EventCollector(this.query, appmapSearchResponse); - const sourceCollector = new SourceCollector(this.vectorTerms, fileSearchResponse); - + const db = new sqlite3(':memory:'); + const fileIndex = await indexFiles(db, this.sourceDirectories); + const fileSearchResults = fileIndex.search(this.vectorTerms.join(' OR ')); let contextCandidate: { results: SearchRpc.SearchResult[]; context: ContextV2.ContextResponse; contextSize: number; }; - let charCount = 0; - let maxEventsPerDiagram = 5; - log(`[search-context] Requested char limit: ${this.charLimit}`); - for (;;) { - log(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`); + try { + const eventsCollector = new EventCollector(this.vectorTerms.join(' '), appmapSearchResponse); + const snippetIndex = await indexSnippets(db, fileSearchResults); - contextCandidate = await eventsCollector.collectEvents( - maxEventsPerDiagram, - this.excludePatterns, - this.includePatterns, - this.includeTypes - ); + let charCount = 0; + let maxEventsPerDiagram = 5; + log(`[search-context] Requested char limit: ${this.charLimit}`); + for (;;) { + log(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`); - const codeSnippetCount = contextCandidate.context.filter( - (item) => item.type === ContextV2.ContextItemType.CodeSnippet - ).length; - let sourceContext: ContextV2.ContextResponse = []; - if (codeSnippetCount === 0) { - sourceContext = await sourceCollector.collectContext(this.charLimit); - } else { - sourceContext = await sourceCollector.collectContext(this.charLimit / 4); - } - contextCandidate.context = contextCandidate.context.concat(sourceContext); + contextCandidate = await eventsCollector.collectEvents( + maxEventsPerDiagram, + this.excludePatterns, + this.includePatterns, + this.includeTypes + ); + + const codeSnippetCount = contextCandidate.context.filter( + (item) => item.type === ContextV2.ContextItemType.CodeSnippet + ).length; + + const charLimit = codeSnippetCount === 0 ? this.charLimit : this.charLimit / 4; + const sourceContext = collectSnippets( + snippetIndex, + this.vectorTerms.join(' OR '), + charLimit + ); + contextCandidate.context = contextCandidate.context.concat(sourceContext); - const appliedContext = applyContext(contextCandidate.context, this.charLimit); - const appliedContextSize = appliedContext.reduce((acc, item) => acc + item.content.length, 0); - contextCandidate.context = appliedContext; - contextCandidate.contextSize = appliedContextSize; - log(`[search-context] Collected an estimated ${appliedContextSize} characters.`); + const appliedContext = applyContext(contextCandidate.context, this.charLimit); + const appliedContextSize = appliedContext.reduce( + (acc, item) => acc + item.content.length, + 0 + ); + contextCandidate.context = appliedContext; + contextCandidate.contextSize = appliedContextSize; + log(`[search-context] Collected an estimated ${appliedContextSize} characters.`); - if (appliedContextSize === charCount || appliedContextSize > this.charLimit) { - break; + if (appliedContextSize === charCount || appliedContextSize > this.charLimit) { + break; + } + charCount = appliedContextSize; + maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5); + log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`); } - charCount = appliedContextSize; - maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5); - log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`); + } finally { + db.close(); } return { diff --git a/packages/cli/src/rpc/explain/SourceCollector.ts b/packages/cli/src/rpc/explain/SourceCollector.ts deleted file mode 100644 index ff5e2bd007..0000000000 --- a/packages/cli/src/rpc/explain/SourceCollector.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { ContextV2 } from '@appland/navie'; -import { FileIndexMatch } from '../../fulltext/FileIndex'; -import withIndex from '../../fulltext/withIndex'; -import { SourceIndexMatch, buildSourceIndex } from '../../fulltext/SourceIndex'; -import { CHARS_PER_SNIPPET } from './collectContext'; - -export default class SourceCollector { - constructor(private keywords: string[], private fileSearchResponse: FileIndexMatch[]) {} - - async collectContext(charLimit: number): Promise { - const sourceIndexDocuments = await withIndex( - 'source', - (indexFileName: string) => buildSourceIndex(indexFileName, this.fileSearchResponse), - (index) => index.search(this.keywords, Math.round(charLimit / CHARS_PER_SNIPPET)) - ); - - const buildLocation = (doc: SourceIndexMatch) => { - return `${doc.fileName}:${doc.from}-${doc.to}`; - }; - - return sourceIndexDocuments.map((doc: SourceIndexMatch) => ({ - directory: doc.directory, - type: ContextV2.ContextItemType.CodeSnippet, - content: doc.content, - location: buildLocation(doc), - })); - } -} diff --git a/packages/cli/src/rpc/explain/collect-snippets.ts b/packages/cli/src/rpc/explain/collect-snippets.ts new file mode 100644 index 0000000000..502894e9c1 --- /dev/null +++ b/packages/cli/src/rpc/explain/collect-snippets.ts @@ -0,0 +1,22 @@ +import { ContextV2 } from '@appland/navie'; +import { SnippetIndex, SnippetSearchResult } from '@appland/search'; +import { CHARS_PER_SNIPPET } from './collectContext'; + +export default function collectSnippets( + snippetIndex: SnippetIndex, + query: string, + charLimit: number +): ContextV2.ContextResponse { + const snippets = snippetIndex.searchSnippets(query, Math.round(charLimit / CHARS_PER_SNIPPET)); + + const buildLocation = (result: SnippetSearchResult) => { + return `${result.filePath}:${result.startLine}-${result.endLine}`; + }; + + return snippets.map((snippet) => ({ + directory: snippet.directory, + type: ContextV2.ContextItemType.CodeSnippet, + content: snippet.content, + location: buildLocation(snippet), + })); +} diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts new file mode 100644 index 0000000000..d4646d8113 --- /dev/null +++ b/packages/cli/src/rpc/explain/index-files.ts @@ -0,0 +1,49 @@ +import sqlite3 from 'better-sqlite3'; +import makeDebug from 'debug'; + +import { + buildFileIndex, + FileIndex, + fileTokens, + isBinaryFile, + isDataFile, + isLargeFile, + listProjectFiles, + readFileSafe, +} from '@appland/search'; + +const debug = makeDebug('appmap:rpc:explain:index-files'); + +const fileFilter = async (path: string) => { + debug('Filtering: %s', path); + if (isBinaryFile(path)) { + debug('Skipping binary file: %s', path); + return false; + } + + const isData = isDataFile(path); + if (isData && (await isLargeFile(path))) { + debug('Skipping large data file: %s', path); + return false; + } + + return true; +}; + +export default async function indexFiles( + db: sqlite3.Database, + directories: string[] +): Promise { + const fileIndex = new FileIndex(db); + + await buildFileIndex( + fileIndex, + directories, + listProjectFiles, + fileFilter, + readFileSafe, + fileTokens + ); + + return fileIndex; +} diff --git a/packages/cli/src/rpc/explain/index-snippets.ts b/packages/cli/src/rpc/explain/index-snippets.ts new file mode 100644 index 0000000000..ceb2d58f01 --- /dev/null +++ b/packages/cli/src/rpc/explain/index-snippets.ts @@ -0,0 +1,21 @@ +import { + buildSnippetIndex, + FileSearchResult, + fileTokens, + langchainSplitter, + readFileSafe, + SnippetIndex, +} from '@appland/search'; +import sqlite3 from 'better-sqlite3'; + +export default async function indexSnippets( + db: sqlite3.Database, + fileSearchResults: FileSearchResult[] +): Promise { + const splitter = langchainSplitter; + + const snippetIndex = new SnippetIndex(db); + await buildSnippetIndex(snippetIndex, fileSearchResults, readFileSafe, splitter, fileTokens); + + return snippetIndex; +} diff --git a/packages/cli/tests/unit/fulltext/FileIndex.spec.ts b/packages/cli/tests/unit/fulltext/FileIndex.spec.ts deleted file mode 100644 index c5f8dcee16..0000000000 --- a/packages/cli/tests/unit/fulltext/FileIndex.spec.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { mkdirSync, readdirSync, writeFileSync } from 'node:fs'; -import fs from 'node:fs/promises'; -import { join } from 'node:path'; - -import sqlite3 from 'better-sqlite3'; -import tmp from 'tmp'; - -import { FileIndex, filterFiles } from '../../../src/fulltext/FileIndex'; -import * as querySymbols from '../../../src/fulltext/querySymbols'; -import { Git, GitState } from '../../../src/telemetry'; -import * as listGitProjectFiles from '../../../src/fulltext/listGitProjectFIles'; - -const originalFsStat: typeof fs = jest.requireActual('node:fs/promises'); -jest.mock('node:fs/promises'); - -describe('FileIndex', () => { - let fileIndex: FileIndex; - let database: sqlite3.Database; - const files: { directory: string; fileName: string }[] = [ - { - directory: 'dir1', - fileName: 'filea', - }, - { - directory: 'dir1', - fileName: 'file11', - }, - { - directory: 'dir1', - fileName: 'file12', - }, - { - directory: 'dir2', - fileName: 'filea', - }, - { - directory: 'dir2', - fileName: 'file21', - }, - { - directory: 'dir2', - fileName: 'file22', - }, - ]; - - beforeEach(() => { - database = new sqlite3(':memory:'); - jest.mocked(fs.stat).mockReset(); - }); - - describe('when matches are found', () => { - beforeEach(() => { - fileIndex = new FileIndex(database); - files.forEach(({ directory, fileName }) => fileIndex.indexFile(directory, fileName)); - }); - afterEach(() => fileIndex.close()); - - it('returns matching file names', () => { - const results = fileIndex.search(['filea'], 10); - expect(results.map((r) => ({ directory: r.directory, fileName: r.fileName }))).toEqual([ - { directory: 'dir1', fileName: 'filea' }, - { directory: 'dir2', fileName: 'filea' }, - ]); - }); - - it('matches alphanumeric file names', () => { - const results = fileIndex.search(['file11'], 10); - expect(results.map((r) => ({ directory: r.directory, fileName: r.fileName }))).toEqual([ - { directory: 'dir1', fileName: 'file11' }, - ]); - }); - - it('does not match directory nanmes', () => { - const results = fileIndex.search(['dir1'], 10); - expect(results.map((r) => ({ directory: r.directory, fileName: r.fileName }))).toEqual([]); - }); - }); - - describe('indexFile', () => { - it('does not query symbols if allowSymbols is false', () => { - const querySymbolsFn = jest.spyOn(querySymbols, 'default'); - const fileIndex = new FileIndex(database); - fileIndex.indexFile('dir1', 'file1', { allowSymbols: false }); - expect(querySymbolsFn).not.toHaveBeenCalled(); - }); - }); - - describe('indexDirectories', () => { - const numFiles = 100; - const batchSize = 50; - const fileNames = Array.from({ length: numFiles }, (_, i) => `file${i}`); - - let fileIndex: FileIndex; - let indexFile: jest.SpyInstance; - let indexDirectory: jest.SpyInstance; - - /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument */ - // `any` is used to avoid type errors and provide access to private methods - - beforeEach(() => { - fileIndex = new FileIndex(database); - indexFile = jest.spyOn(fileIndex, 'indexFile'); - indexDirectory = jest.spyOn(fileIndex as any, 'indexDirectory'); - - jest.spyOn(Git, 'state').mockResolvedValue(GitState.Ok); - jest.mocked(fs.stat).mockResolvedValue({ - isFile: jest.fn().mockReturnValue(true), - size: BigInt(100), - } as any); - jest.spyOn(listGitProjectFiles, 'default').mockResolvedValue(fileNames); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - it('indexes all files in batches', async () => { - await fileIndex.indexDirectories(['dir1'], undefined, undefined, batchSize); - - expect(indexDirectory).toHaveBeenCalledTimes(numFiles / batchSize); - expect(indexFile).toHaveBeenCalledTimes(numFiles); - expect(indexFile.mock.calls.map(([, fileName]) => fileName)).toEqual(fileNames); - }); - - describe('when some are filtered out', () => { - describe('by excludePattern', () => { - it('removes them', async () => { - await fileIndex.indexDirectories(['dir1'], [/file1/], undefined); - - expect(fileIndex.search(['file1'], 10)).toHaveLength(0); - expect(fileIndex.search(['file2'], 10)).toHaveLength(1); - }); - }); - - describe('by includePattern', () => { - it('removes them', async () => { - await fileIndex.indexDirectories(['dir1'], undefined, [/file1/]); - - expect(fileIndex.search(['file1'], 10)).toHaveLength(1); - expect(fileIndex.search(['file2'], 10)).toHaveLength(0); - }); - }); - }); - - /* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument */ - }); -}); - -describe(filterFiles, () => { - beforeEach(() => { - fs.stat = originalFsStat.stat; - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - it('filters out binary files, non-files, and large data files', async () => { - const dir = tmp.dirSync({ unsafeCleanup: true }).name; - writeFileSync(join(dir, 'file.txt'), 'hello'); - writeFileSync(join(dir, 'file.zip'), 'hello'); - writeFileSync(join(dir, 'file.json'), 'hello'); - writeFileSync(join(dir, 'large.txt'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.js'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.ts'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.haml'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.java'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.mjs'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.pyc'), Buffer.alloc(100_000)); - writeFileSync(join(dir, 'large.json'), Buffer.alloc(100_000)); - mkdirSync(join(dir, 'dir')); - - const fileList = readdirSync(dir); - const filtered = await filterFiles(dir, fileList); - expect(filtered).toEqual([ - 'file.json', - 'file.txt', - 'large.haml', - 'large.java', - 'large.js', - 'large.mjs', - 'large.ts', - 'large.txt', - ]); - }); -}); diff --git a/packages/cli/tests/unit/fulltext/SourceIndex.spec.ts b/packages/cli/tests/unit/fulltext/SourceIndex.spec.ts deleted file mode 100644 index 266043dfc8..0000000000 --- a/packages/cli/tests/unit/fulltext/SourceIndex.spec.ts +++ /dev/null @@ -1,52 +0,0 @@ -import sqlite3 from 'better-sqlite3'; -import { SourceIndex, SourceIndexDocument } from '../../../src/fulltext/SourceIndex'; -import queryKeywords from '../../../src/fulltext/queryKeywords'; - -describe('SourceIndex', () => { - let sourceIndex: SourceIndex; - let database: sqlite3.Database; - - beforeEach(() => (database = new sqlite3(':memory:'))); - - describe('when matches are found', () => { - const files: SourceIndexDocument[] = [ - { - directory: __dirname, - fileName: 'FileIndex.spec.ts', - }, - { - directory: __dirname, - fileName: 'SourceIndex.spec.ts', - }, - ]; - - beforeEach(async () => { - sourceIndex = new SourceIndex(database); - await sourceIndex.indexFiles(files); - }); - afterEach(() => sourceIndex.close()); - - it('returns matching source code snippets', () => { - const indexWords = ['index', 'source']; - - const results = sourceIndex.search(['SourceIndex'], 10); - expect(results.length).toBeTruthy(); - - for (const result of results) { - if ( - !queryKeywords([result.directory, result.fileName, result.content]).some((keyword) => - indexWords.includes(keyword) - ) - ) { - throw new Error( - `Expected result content (${queryKeywords([ - result.directory, - result.fileName, - result.content, - ]).join(', ')}) to contain one of ${indexWords.join(', ')}` - ); - } - } - }); - }); -}); diff --git a/packages/cli/tests/unit/fulltext/listGitProjectFiles.spec.ts b/packages/cli/tests/unit/fulltext/listGitProjectFiles.spec.ts deleted file mode 100644 index 3cbc2f94d0..0000000000 --- a/packages/cli/tests/unit/fulltext/listGitProjectFiles.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import * as utilsModule from '../../../src/utils'; -import listGitProjectFiles from '../../../src/fulltext/listGitProjectFIles'; -import * as childProcess from 'node:child_process'; -import type { ChildProcess } from 'node:child_process'; - -// Mock dependencies -jest.mock('node:child_process'); -jest.mock('../../../src/utils', () => ({ - ...jest.requireActual('../../../src/utils'), - isFile: jest.fn(), -})); - -describe('listGitProjectFiles', () => { - const mockDirectory = '/mock/directory'; - - beforeEach(() => { - // Clear all mocks before each test - jest.clearAllMocks(); - // Setup default return values for mocked functions - jest.mocked(childProcess.exec).mockImplementation((cmd: string, _options, callback) => { - const cb = callback as (_: unknown, __: unknown) => void; - if (cmd.includes('ls-files')) { - cb(null, { stdout: 'file1.js' }); - } else if (cmd.includes('status --porcelain')) { - cb(null, { - stdout: ` -?? newFile.ts -M stagedFile.js - M unstagedFile.js -`, - }); - } else { - cb(null, { stdout: '' }); - } - - return {} as ChildProcess; - }); - jest.mocked(utilsModule.isFile).mockResolvedValue(true); - }); - - it('includes git-managed files, including modified files, as well as untracked', async () => { - const files = await listGitProjectFiles(mockDirectory); - - expect(files.length).toBe(4); - expect(files).toContain('file1.js'); - expect(files).toContain('newFile.ts'); - expect(files).toContain('stagedFile.js'); - expect(files).toContain('unstagedFile.js'); - }); -}); diff --git a/packages/cli/tests/unit/fulltext/listProjectFiles.spec.ts b/packages/cli/tests/unit/fulltext/listProjectFiles.spec.ts deleted file mode 100644 index 712224a0a4..0000000000 --- a/packages/cli/tests/unit/fulltext/listProjectFiles.spec.ts +++ /dev/null @@ -1,60 +0,0 @@ -import listProjectFiles from '../../../src/fulltext/listProjectFiles'; -import * as fs from 'fs'; -import * as fsp from 'fs/promises'; -import { join } from 'path'; - -jest.mock('fs/promises', () => ({ - readdir: jest.fn(), -})); - -describe('listProjectFiles', () => { - beforeEach(() => { - jest.resetAllMocks(); - }); - - it('lists all files in a directory', async () => { - const mockFiles = [ - { name: 'index.js', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, - { name: 'logo.png', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, - { name: 'utils', isFile: () => false, isDirectory: () => true } as unknown as fs.Dirent, - // Pretend utils directory contains a single file - { name: 'helper.js', isFile: () => true, isDirectory: () => false } as unknown as fs.Dirent, - ]; - - const baseDir = join('fake', 'directory'); - jest.mocked(fsp.readdir).mockImplementation((dir: fs.PathLike) => { - if (dir.toString() === join(baseDir, 'utils')) return Promise.resolve([mockFiles[3]]); - else if (dir.toString() === baseDir) return Promise.resolve(mockFiles.slice(0, 3)); - else return Promise.resolve([]); - }); - - const files = await listProjectFiles(baseDir); - expect(files).toContain('index.js'); - expect(files).toContain(join('utils', 'helper.js')); - expect(fsp.readdir).toHaveBeenCalledTimes(2); // baseDir + utils - }); - - it('ignores directories specified in IGNORE_DIRECTORIES', async () => { - const mockDirectories = [ - { - name: 'node_modules', - isFile: () => false, - isDirectory: () => true, - } as unknown as fs.Dirent, - { name: 'src', isFile: () => false, isDirectory: () => true } as unknown as fs.Dirent, - ]; - - const directoriesRead = new Array(); - const baseDir = join('fake', 'directory'); - jest.mocked(fsp.readdir).mockImplementation((dir: fs.PathLike) => { - directoriesRead.push(dir.toString()); - - if (dir.toString() === baseDir) return Promise.resolve(mockDirectories); - else return Promise.resolve([]); - }); - - const files = await listProjectFiles(baseDir); - expect(files).toEqual([]); - expect(directoriesRead).toEqual([baseDir, join(baseDir, 'src')]); - }); -}); diff --git a/packages/cli/tests/unit/fulltext/queryKeywords.spec.ts b/packages/cli/tests/unit/fulltext/queryKeywords.spec.ts deleted file mode 100644 index c112ba3c67..0000000000 --- a/packages/cli/tests/unit/fulltext/queryKeywords.spec.ts +++ /dev/null @@ -1,83 +0,0 @@ -import queryKeywords from '../../../src/fulltext/queryKeywords'; - -describe('queryKeywords', () => { - it('should sanitize and split keywords correctly', () => { - const input = 'Hello_World! Welcome to the universe.'; - expect(queryKeywords(input)).toEqual(['hello', 'world', 'helloworld', 'welcome', 'universe']); - }); - - it('should filter out stop words', () => { - const input = 'the quick brown fox jumps over the lazy dog'; - expect(queryKeywords(input)).toEqual(['quick', 'brown', 'fox', 'jumps', 'lazy', 'dog']); - }); - - it('should handle under_score words', () => { - const input = 'hello_world this_is_a_test'; - expect(queryKeywords(input)).toEqual([ - 'hello', - 'world', - 'helloworld', - 'this', - 'thisis', - 'isa', - 'test', - 'atest', - ]); - }); - - it('should handle camelCased words', () => { - const input = 'helloWorld thisIsATest'; - expect(queryKeywords(input)).toEqual([ - 'hello', - 'world', - 'helloworld', - 'this', - 'thisis', - 'isa', - 'test', - 'atest', - ]); - }); - - it('should handle array input', () => { - const input = ['hello', 'world']; - expect(queryKeywords(input)).toEqual(['hello', 'world']); - }); - - it('should return empty array for invalid input', () => { - expect(queryKeywords(undefined)).toEqual([]); - expect(queryKeywords('')).toEqual([]); - }); - - it('should only return words with length >= 2', () => { - const input = 'a an i am'; - expect(queryKeywords(input)).toEqual(['am']); - }); - - it('should split camelCased words with consecutive uppercase letters', () => { - const input = 'dataForUSACounties'; - expect(queryKeywords(input)).toEqual([ - 'data', - 'datafor', - 'usa', - 'forusa', - 'counties', - 'usacounties', - ]); - }); - - it('should handle strings with multiple camelizations', () => { - const input = 'XMLHttpRequest and HTTPResponseCode'; - expect(queryKeywords(input)).toEqual([ - 'xml', - 'http', - 'xmlhttp', - 'request', - 'httprequest', - 'http', - 'response', - 'httpresponse', - 'responsecode', - ]); - }); -}); diff --git a/packages/cli/tests/unit/fulltext/querySymbols.spec.ts b/packages/cli/tests/unit/fulltext/querySymbols.spec.ts deleted file mode 100644 index a2504125cb..0000000000 --- a/packages/cli/tests/unit/fulltext/querySymbols.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { join } from 'path'; -import querySymbols from '../../../src/fulltext/querySymbols'; - -describe('querySymbols', () => { - describe('csharp', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.cs'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - [ - 'ClassOne', - 'ClassTwo', - 'ClassThree', - 'ClassFour', - 'StructOne', - 'StructTwo', - 'StructThree', - 'IOne', - 'ITwo', - 'IThree', - 'IFour', - 'Example', - 'Season', - 'ErrorCode', - 'ClassWithMethods', - 'ClassWithMethods', - '~ClassWithMethods', - 'MethodOne', - 'MethodTwo', - 'MethodThree', - 'MethodFour', - ].sort() - ); - }); - }); - - describe('c/cpp', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.c'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['foo', 'main', 'MyStruct', 'MyOtherStruct', 'Point'].sort() - ); - }); - }); - - describe('rust', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.rs'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['Foo', 'new', 'MyUnion', 'MyTrait', 'my_function', 'Point', 'main', 'Москва'].sort() - ); - }); - }); - - describe('go', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.go'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - [ - 'Locker', - 'MyInterface', - 'LockerImpl', - 'Lock', - 'Unlock', - 'Reader', - 'ReadWriter', - 'main', - 'unicodeβ', - ].sort() - ); - }); - }); - - describe('ruby', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.rb'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['MyClass', 'MyModule', 'my_method', 'my_module_method', 'some_function'].sort() - ); - }); - }); - - describe('python', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.py'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['MyClass', '__init__', 'say_hello', 'some_function'].sort() - ); - }); - }); - - describe('java', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.java'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - [ - 'SampleEnum', - 'SampleAnnotation', - 'ISample', - 'Sample', - 'main', - 'performUserOperation', - ].sort() - ); - }); - }); - - describe('javascript/typescript', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.ts'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - [ - 'MyClass', - 'constructor', - '#myClassMethod', - 'myFunction', - 'myOtherFunction', - 'MyType', - 'MyInterface', - 'MyEnum', - 'myObjectMethod', - 'newLineBraces', - '$lineBreak', - ].sort() - ); - }); - }); - - describe('kotlin', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.kt'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['main', 'Rectangle', 'Predicate', 'Color', 'filter', 'sort'].sort() - ); - }); - }); - - describe('php', () => { - it('identifies symbols', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.php'); - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - ['MyClass', 'myMethod', 'Talk', 'talk', 'myFunction'].sort() - ); - }); - }); - - describe('generic', () => { - const srcPath = join(__dirname, '../fixtures/source/sample.generic'); - - it('identifies symbols', () => { - const symbols = querySymbols(srcPath); - expect(symbols.sort()).toStrictEqual( - [ - 'fibonacci', - 'Spacecraft', - 'Spacecraft', - 'describe', - 'PlanetType', - 'Resolution', - 'VideoMode', - 'Counter', - 'increment', - 'increment', - 'reset', - 'Something', - '_init', - 'other_something', - 'something', - 'perform_action', - ].sort() - ); - }); - - it('does not run if allowGeneric is false', () => { - const symbols = querySymbols(srcPath, false); - expect(symbols).toStrictEqual([]); - }); - }); -}); diff --git a/packages/cli/tests/unit/rpc/explain/SearchContextCollector.spec.ts b/packages/cli/tests/unit/rpc/explain/SearchContextCollector.spec.ts deleted file mode 100644 index d2719cc860..0000000000 --- a/packages/cli/tests/unit/rpc/explain/SearchContextCollector.spec.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { SearchRpc } from '@appland/rpc'; -import { ContextCollector } from '../../../../src/rpc/explain/collectContext'; -import AppMapIndex from '../../../../src/fulltext/AppMapIndex'; -import * as withIndex from '../../../../src/fulltext/withIndex'; -import * as navie from '@appland/navie'; -import EventCollector from '../../../../src/rpc/explain/EventCollector'; - -jest.mock('../../../../src/fulltext/AppMapIndex'); -jest.mock('@appland/navie'); - -describe('SearchContextCollector', () => { - const vectorTerms = ['login', 'user']; - const charLimit = 5000; - let contextCollector: ContextCollector; - - beforeEach(() => { - jest.mocked(navie.applyContext).mockImplementation((context) => context); - }); - afterEach(() => jest.restoreAllMocks()); - - describe('collectContext', () => { - describe('appmaps', () => { - beforeEach(() => { - contextCollector = new ContextCollector(['a', 'b'], [], vectorTerms, charLimit); - }); - - it('returns context for specified appmaps', async () => { - const mockAppmaps = ['appmap1', 'appmap2']; - contextCollector.appmaps = mockAppmaps; - - const mockContext: navie.ContextV2.ContextResponse = [ - { - type: navie.ContextV2.ContextItemType.SequenceDiagram, - content: 'diagram1', - }, - { - type: navie.ContextV2.ContextItemType.SequenceDiagram, - content: 'diagram2', - }, - ]; - - AppMapIndex.search = jest.fn().mockRejectedValue(new Error('Unexpected call to search')); - - EventCollector.prototype.collectEvents = jest.fn().mockResolvedValue({ - results: [], - context: mockContext, - contextSize: 4545, - }); - - const collectedContext = await contextCollector.collectContext(); - - expect(collectedContext.searchResponse.numResults).toBe(mockAppmaps.length); - expect(collectedContext.context).toEqual(mockContext); - }); - - it('handles search across all appmaps', async () => { - const mockSearchResponse: SearchRpc.SearchResponse = { - numResults: 10, - results: [ - { - appmap: 'appmap1', - directory: 'a', - score: 1, - events: [{ fqid: 'function:1', score: 1, eventIds: [1, 2] }], - }, - { - appmap: 'appmap2', - directory: 'a', - score: 1, - events: [{ fqid: 'function:2', score: 1, eventIds: [3, 4] }], - }, - { - appmap: 'appmap3', - directory: 'b', - score: 1, - events: [{ fqid: 'function:3', score: 1, eventIds: [5, 6] }], - }, - ], - }; - - AppMapIndex.search = jest.fn().mockResolvedValue(mockSearchResponse); - - const mockContext: navie.ContextV2.ContextResponse = [ - { - type: navie.ContextV2.ContextItemType.SequenceDiagram, - content: 'diagram1', - }, - ]; - - EventCollector.prototype.collectEvents = jest.fn().mockResolvedValue({ - results: [], - context: mockContext, - contextSize: 3000, - }); - - const collectedContext = await contextCollector.collectContext(); - - expect(AppMapIndex.search).toHaveBeenCalledWith(['a', 'b'], vectorTerms.join(' '), { - maxResults: expect.any(Number), - }); - expect(collectedContext.searchResponse.numResults).toBe(10); - expect(collectedContext.context).toEqual(mockContext); - }); - }); - - describe('with empty vector terms', () => { - it('returns an empty context', async () => { - const indexSearch = jest.spyOn(withIndex, 'default'); - const emptyVectorTerms = ['', ' ']; - - const contextCollector = new ContextCollector( - ['example'], - ['src'], - emptyVectorTerms, - charLimit - ); - const result = await contextCollector.collectContext(); - expect(result).toStrictEqual({ - searchResponse: { - results: [], - numResults: 0, - }, - context: [], - }); - - expect(indexSearch).not.toHaveBeenCalled(); - }); - }); - }); -}); diff --git a/packages/cli/tests/unit/rpc/explain/collectContext.spec.ts b/packages/cli/tests/unit/rpc/explain/collectContext.spec.ts new file mode 100644 index 0000000000..5120f4d8d2 --- /dev/null +++ b/packages/cli/tests/unit/rpc/explain/collectContext.spec.ts @@ -0,0 +1,124 @@ +import { SearchRpc } from '@appland/rpc'; +import { ContextCollector } from '../../../../src/rpc/explain/collectContext'; +import AppMapIndex from '../../../../src/fulltext/AppMapIndex'; +import * as navie from '@appland/navie'; +import EventCollector from '../../../../src/rpc/explain/EventCollector'; + +jest.mock('../../../../src/fulltext/AppMapIndex'); +jest.mock('@appland/navie'); + +describe('collectContext', () => { + const vectorTerms = ['login', 'user']; + const charLimit = 5000; + let contextCollector: ContextCollector; + + beforeEach(() => { + jest.mocked(navie.applyContext).mockImplementation((context) => context); + }); + afterEach(() => jest.restoreAllMocks()); + + describe('appmaps', () => { + beforeEach(() => { + contextCollector = new ContextCollector(['a', 'b'], [], vectorTerms, charLimit); + }); + + it('returns context for specified appmaps', async () => { + const mockAppmaps = ['appmap1', 'appmap2']; + contextCollector.appmaps = mockAppmaps; + + const mockContext: navie.ContextV2.ContextResponse = [ + { + type: navie.ContextV2.ContextItemType.SequenceDiagram, + content: 'diagram1', + }, + { + type: navie.ContextV2.ContextItemType.SequenceDiagram, + content: 'diagram2', + }, + ]; + + AppMapIndex.search = jest.fn().mockRejectedValue(new Error('Unexpected call to search')); + + EventCollector.prototype.collectEvents = jest.fn().mockResolvedValue({ + results: [], + context: mockContext, + contextSize: 4545, + }); + + const collectedContext = await contextCollector.collectContext(); + + expect(collectedContext.searchResponse.numResults).toBe(mockAppmaps.length); + expect(collectedContext.context).toEqual(mockContext); + }); + + it('handles search across all appmaps', async () => { + const mockSearchResponse: SearchRpc.SearchResponse = { + numResults: 10, + results: [ + { + appmap: 'appmap1', + directory: 'a', + score: 1, + events: [{ fqid: 'function:1', score: 1, eventIds: [1, 2] }], + }, + { + appmap: 'appmap2', + directory: 'a', + score: 1, + events: [{ fqid: 'function:2', score: 1, eventIds: [3, 4] }], + }, + { + appmap: 'appmap3', + directory: 'b', + score: 1, + events: [{ fqid: 'function:3', score: 1, eventIds: [5, 6] }], + }, + ], + }; + + AppMapIndex.search = jest.fn().mockResolvedValue(mockSearchResponse); + + const mockContext: navie.ContextV2.ContextResponse = [ + { + type: navie.ContextV2.ContextItemType.SequenceDiagram, + content: 'diagram1', + }, + ]; + + EventCollector.prototype.collectEvents = jest.fn().mockResolvedValue({ + results: [], + context: mockContext, + contextSize: 3000, + }); + + const collectedContext = await contextCollector.collectContext(); + + expect(AppMapIndex.search).toHaveBeenCalledWith(['a', 'b'], vectorTerms.join(' '), { + maxResults: expect.any(Number), + }); + expect(collectedContext.searchResponse.numResults).toBe(10); + expect(collectedContext.context).toEqual(mockContext); + }); + }); + + describe('with empty vector terms', () => { + it('returns an empty context', async () => { + const emptyVectorTerms = ['', ' ']; + + const contextCollector = new ContextCollector( + ['example'], + ['src'], + emptyVectorTerms, + charLimit + ); + const result = await contextCollector.collectContext(); + expect(result).toStrictEqual({ + searchResponse: { + results: [], + numResults: 0, + }, + context: [], + }); + }); + }); +}); From 7d509e89e396c7332c8a41f47a92583067e5e13d Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 25 Oct 2024 11:53:36 -0400 Subject: [PATCH 10/15] chore: Update eslint rules --- packages/search/.eslintrc.js | 47 -- packages/search/eslint.config.mjs | 68 ++ packages/search/package.json | 16 +- yarn.lock | 1278 +++++++++++++++++++++++++---- 4 files changed, 1189 insertions(+), 220 deletions(-) delete mode 100644 packages/search/.eslintrc.js create mode 100644 packages/search/eslint.config.mjs diff --git a/packages/search/.eslintrc.js b/packages/search/.eslintrc.js deleted file mode 100644 index d98123a73e..0000000000 --- a/packages/search/.eslintrc.js +++ /dev/null @@ -1,47 +0,0 @@ -/* eslint-disable eslint-comments/disable-enable-pair */ -/* eslint-disable @typescript-eslint/no-var-requires */ -const path = require('path'); - -module.exports = { - plugins: ['@typescript-eslint', 'eslint-comments', 'jest', 'promise', 'unicorn'], - extends: [ - 'plugin:@typescript-eslint/recommended', - 'plugin:@typescript-eslint/recommended-requiring-type-checking', - 'plugin:eslint-comments/recommended', - 'plugin:jest/recommended', - 'plugin:promise/recommended', - 'prettier', - ], - env: { - node: true, - browser: true, - jest: true, - }, - parserOptions: { - project: path.join(__dirname, 'tsconfig.json'), - }, - root: true, - rules: { - 'no-param-reassign': ['error', { props: false }], - 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', - 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', - '@typescript-eslint/lines-between-class-members': 'off', - 'no-restricted-syntax': 'off', - 'import/no-cycle': 'off', - 'prettier/prettier': ['error'], - 'max-classes-per-file': 'off', - }, - overrides: [ - { - files: ['*.js'], - extends: ['eslint:recommended', 'prettier'], - parserOptions: { - project: path.join(__dirname, 'jsconfig.json'), - }, - plugins: ['prettier'], - rules: { - 'unicorn/prefer-module': 'off', - }, - }, - ], -}; diff --git a/packages/search/eslint.config.mjs b/packages/search/eslint.config.mjs new file mode 100644 index 0000000000..d5785f201c --- /dev/null +++ b/packages/search/eslint.config.mjs @@ -0,0 +1,68 @@ +import typescriptEslint from '@typescript-eslint/eslint-plugin'; +import eslintComments from 'eslint-plugin-eslint-comments'; +import jest from 'eslint-plugin-jest'; +import promise from 'eslint-plugin-promise'; +import globals from 'globals'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all, +}); + +export default [ + ...compat.extends( + 'plugin:@typescript-eslint/recommended', + 'plugin:@typescript-eslint/recommended-requiring-type-checking', + 'plugin:eslint-comments/recommended', + 'plugin:jest/recommended', + 'plugin:promise/recommended', + 'plugin:prettier/recommended' + ), + { + plugins: { + '@typescript-eslint': typescriptEslint, + 'eslint-comments': eslintComments, + jest, + promise, + }, + + languageOptions: { + globals: { + ...globals.node, + ...globals.browser, + ...globals.jest, + }, + + ecmaVersion: 5, + sourceType: 'commonjs', + + parserOptions: { + project: '/home/db/dev/applandinc/appmap-js/packages/search/tsconfig.json', + }, + }, + + rules: { + 'no-param-reassign': [ + 'error', + { + props: false, + }, + ], + + 'no-console': 'off', + 'no-debugger': 'off', + '@typescript-eslint/lines-between-class-members': 'off', + 'no-restricted-syntax': 'off', + 'import/no-cycle': 'off', + 'prettier/prettier': ['error'], + 'max-classes-per-file': 'off', + }, + }, +]; diff --git a/packages/search/package.json b/packages/search/package.json index d0b2459214..e7870a287a 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -21,25 +21,27 @@ "author": "AppLand, Inc", "license": "Commons Clause + MIT", "devDependencies": { - "@eslint/js": "~8.57.0", + "@eslint/eslintrc": "^3.1.0", + "@eslint/js": "^9.13.0", "@types/better-sqlite3": "^7.6.9", "@types/jest": "^29.5.4", "@types/jest-sinon": "^1.0.2", "@types/node": "^16", "esbuild": "0.19.8", - "eslint": "^8.56.0", - "eslint-config-prettier": "^8.3.0", + "eslint": "^9", + "eslint-config-prettier": "^9", "eslint-plugin-eslint-comments": "^3.2.0", - "eslint-plugin-import": "^2.22.1", + "eslint-plugin-import": "^2.31.0", "eslint-plugin-jest": "^28.8.3", + "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-promise": "^7.1.0", - "eslint-plugin-unicorn": "latest", + "globals": "^15.11.0", "jest": "^29.7.0", "jest-sinon": "^1.1.0", "lint-staged": "^10.5.4", "memfs": "^3.4.13", "node-fetch": "2.6.7", - "prettier": "^2.7.1", + "prettier": "^3.3.3", "sinon": "^11.1.2", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", @@ -47,7 +49,7 @@ "tsc": "^2.0.3", "type-fest": "^3.1.0", "typescript": "^4.9.5", - "typescript-eslint": "^7.7.0" + "typescript-eslint": "^8.11.0" }, "dependencies": { "better-sqlite3": "^9.5.0", diff --git a/yarn.lock b/yarn.lock index 714c9ab774..ed0cffb34f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -590,27 +590,29 @@ __metadata: version: 0.0.0-use.local resolution: "@appland/search@workspace:packages/search" dependencies: - "@eslint/js": ~8.57.0 + "@eslint/eslintrc": ^3.1.0 + "@eslint/js": ^9.13.0 "@types/better-sqlite3": ^7.6.9 "@types/jest": ^29.5.4 "@types/jest-sinon": ^1.0.2 "@types/node": ^16 better-sqlite3: ^9.5.0 esbuild: 0.19.8 - eslint: ^8.56.0 - eslint-config-prettier: ^8.3.0 + eslint: ^9 + eslint-config-prettier: ^9 eslint-plugin-eslint-comments: ^3.2.0 - eslint-plugin-import: ^2.22.1 + eslint-plugin-import: ^2.31.0 eslint-plugin-jest: ^28.8.3 + eslint-plugin-prettier: ^5.2.1 eslint-plugin-promise: ^7.1.0 - eslint-plugin-unicorn: latest + globals: ^15.11.0 jest: ^29.7.0 jest-sinon: ^1.1.0 lint-staged: ^10.5.4 memfs: ^3.4.13 node-fetch: 2.6.7 package.json: ^2.0.1 - prettier: ^2.7.1 + prettier: ^3.3.3 sinon: ^11.1.2 ts-jest: ^29.0.5 ts-node: ^10.9.1 @@ -618,7 +620,7 @@ __metadata: tsc: ^2.0.3 type-fest: ^3.1.0 typescript: ^4.9.5 - typescript-eslint: ^7.7.0 + typescript-eslint: ^8.11.0 yargs: ^17.7.2 bin: search: built/cli.js @@ -1891,13 +1893,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-validator-identifier@npm:^7.24.7": - version: 7.25.7 - resolution: "@babel/helper-validator-identifier@npm:7.25.7" - checksum: 062f55208deead4876eb474dc6fd55155c9eada8d0a505434de3b9aa06c34195562e0f3142b22a08793a38d740238efa2fe00ff42956cdcb8ac03f0b6c542247 - languageName: node - linkType: hard - "@babel/helper-validator-option@npm:^7.16.7": version: 7.16.7 resolution: "@babel/helper-validator-option@npm:7.16.7" @@ -5756,6 +5751,31 @@ __metadata: languageName: node linkType: hard +"@eslint-community/regexpp@npm:^4.11.0": + version: 4.11.1 + resolution: "@eslint-community/regexpp@npm:4.11.1" + checksum: 6986685529d30e33c2640973c3d8e7ddd31bef3cc8cb10ad54ddc1dea12680779a2c23a45562aa1462c488137a3570e672d122fac7da22d82294382d915cec70 + languageName: node + linkType: hard + +"@eslint/config-array@npm:^0.18.0": + version: 0.18.0 + resolution: "@eslint/config-array@npm:0.18.0" + dependencies: + "@eslint/object-schema": ^2.1.4 + debug: ^4.3.1 + minimatch: ^3.1.2 + checksum: 5ff748e1788745bfb3160c3b3151d62a7c054e336e9fe8069e86cfa6106f3abbd59b24f1253122268295f98c66803e9a7b23d7f947a8c00f62d2060cc44bc7fc + languageName: node + linkType: hard + +"@eslint/core@npm:^0.7.0": + version: 0.7.0 + resolution: "@eslint/core@npm:0.7.0" + checksum: 91d4aa2805f356fb0bba693411deab91590472666e22c9c03304ba03b288b74403a5e120db16d0926ea94281e15563a8d4d519cd1e565d514e2d5015a84b8575 + languageName: node + linkType: hard + "@eslint/eslintrc@npm:^0.4.3": version: 0.4.3 resolution: "@eslint/eslintrc@npm:0.4.3" @@ -5807,6 +5827,23 @@ __metadata: languageName: node linkType: hard +"@eslint/eslintrc@npm:^3.1.0": + version: 3.1.0 + resolution: "@eslint/eslintrc@npm:3.1.0" + dependencies: + ajv: ^6.12.4 + debug: ^4.3.2 + espree: ^10.0.1 + globals: ^14.0.0 + ignore: ^5.2.0 + import-fresh: ^3.2.1 + js-yaml: ^4.1.0 + minimatch: ^3.1.2 + strip-json-comments: ^3.1.1 + checksum: b0a9bbd98c8b9e0f4d975b042ff9b874dde722b20834ea2ff46551c3de740d4f10f56c449b790ef34d7f82147cbddfc22b004a43cc885dbc2664bb134766b5e4 + languageName: node + linkType: hard + "@eslint/js@npm:8.57.0, @eslint/js@npm:~8.57.0": version: 8.57.0 resolution: "@eslint/js@npm:8.57.0" @@ -5814,6 +5851,29 @@ __metadata: languageName: node linkType: hard +"@eslint/js@npm:9.13.0, @eslint/js@npm:^9.13.0": + version: 9.13.0 + resolution: "@eslint/js@npm:9.13.0" + checksum: ad5dd72aa75bd8d5bd3c1ffe68cf748aed7edef5fcf97193eb52af35dbb89a1999f526a0e2c169ef5572afbbbbb5f37d6fd0af2991d9ccdc29f753da5cc0f532 + languageName: node + linkType: hard + +"@eslint/object-schema@npm:^2.1.4": + version: 2.1.4 + resolution: "@eslint/object-schema@npm:2.1.4" + checksum: 5a03094115bcdab7991dbbc5d17a9713f394cebb4b44d3eaf990d7487b9b8e1877b817997334ab40be52e299a0384595c6f6ba91b389901e5e1d21efda779271 + languageName: node + linkType: hard + +"@eslint/plugin-kit@npm:^0.2.0": + version: 0.2.1 + resolution: "@eslint/plugin-kit@npm:0.2.1" + dependencies: + levn: ^0.4.1 + checksum: 3c4fc57ff1fa8f08c9c7c3f2272883fe455faf21e352bd5c7965406d60b366ae8eb913fd53e846a688eaa9eca7f8100428361ac598bd56499fced6991d7b200e + languageName: node + linkType: hard + "@fal-works/esbuild-plugin-global-externals@npm:^2.1.2": version: 2.1.2 resolution: "@fal-works/esbuild-plugin-global-externals@npm:2.1.2" @@ -5942,6 +6002,23 @@ __metadata: languageName: node linkType: hard +"@humanfs/core@npm:^0.19.0": + version: 0.19.0 + resolution: "@humanfs/core@npm:0.19.0" + checksum: d73c153e9a41efce401cdf8eaa831e5b01630b45a46678eded3803347251a24446f1500b0074750fcab0a88d947609b164a0d5bba57f58ec18167bea01c69ac5 + languageName: node + linkType: hard + +"@humanfs/node@npm:^0.16.5": + version: 0.16.5 + resolution: "@humanfs/node@npm:0.16.5" + dependencies: + "@humanfs/core": ^0.19.0 + "@humanwhocodes/retry": ^0.3.0 + checksum: ae4799c6bf436450e1b1836f23fdb4ce0eb862df8e02fd498ee7d8ebe552d85fe36ccac81fcfbe39bf43cb49b302ae438d94699a451d1cfc78f64198d4b45674 + languageName: node + linkType: hard + "@humanwhocodes/config-array@npm:^0.11.14": version: 0.11.14 resolution: "@humanwhocodes/config-array@npm:0.11.14" @@ -5996,6 +6073,13 @@ __metadata: languageName: node linkType: hard +"@humanwhocodes/retry@npm:^0.3.0, @humanwhocodes/retry@npm:^0.3.1": + version: 0.3.1 + resolution: "@humanwhocodes/retry@npm:0.3.1" + checksum: 7e5517bb51dbea3e02ab6cacef59a8f4b0ca023fc4b0b8cbc40de0ad29f46edd50b897c6e7fba79366a0217e3f48e2da8975056f6c35cfe19d9cc48f1d03c1dd + languageName: node + linkType: hard + "@intervolga/optimize-cssnano-plugin@npm:^1.0.5": version: 1.0.6 resolution: "@intervolga/optimize-cssnano-plugin@npm:1.0.6" @@ -8087,6 +8171,13 @@ __metadata: languageName: node linkType: hard +"@pkgr/core@npm:^0.1.0": + version: 0.1.1 + resolution: "@pkgr/core@npm:0.1.1" + checksum: 6f25fd2e3008f259c77207ac9915b02f1628420403b2630c92a07ff963129238c9262afc9e84344c7a23b5cc1f3965e2cd17e3798219f5fd78a63d144d3cceba + languageName: node + linkType: hard + "@radix-ui/number@npm:1.0.1": version: 1.0.1 resolution: "@radix-ui/number@npm:1.0.1" @@ -8776,6 +8867,13 @@ __metadata: languageName: node linkType: hard +"@rtsao/scc@npm:^1.1.0": + version: 1.1.0 + resolution: "@rtsao/scc@npm:1.1.0" + checksum: 17d04adf404e04c1e61391ed97bca5117d4c2767a76ae3e879390d6dec7b317fcae68afbf9e98badee075d0b64fa60f287729c4942021b4d19cd01db77385c01 + languageName: node + linkType: hard + "@rushstack/eslint-patch@npm:^1.2.0": version: 1.2.0 resolution: "@rushstack/eslint-patch@npm:1.2.0" @@ -11003,6 +11101,13 @@ __metadata: languageName: node linkType: hard +"@types/estree@npm:^1.0.6": + version: 1.0.6 + resolution: "@types/estree@npm:1.0.6" + checksum: 8825d6e729e16445d9a1dd2fb1db2edc5ed400799064cd4d028150701031af012ba30d6d03fe9df40f4d7a437d0de6d2b256020152b7b09bde9f2e420afdffd9 + languageName: node + linkType: hard + "@types/events@npm:^3.0.3": version: 3.0.3 resolution: "@types/events@npm:3.0.3" @@ -11990,6 +12095,29 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/eslint-plugin@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/eslint-plugin@npm:8.11.0" + dependencies: + "@eslint-community/regexpp": ^4.10.0 + "@typescript-eslint/scope-manager": 8.11.0 + "@typescript-eslint/type-utils": 8.11.0 + "@typescript-eslint/utils": 8.11.0 + "@typescript-eslint/visitor-keys": 8.11.0 + graphemer: ^1.4.0 + ignore: ^5.3.1 + natural-compare: ^1.4.0 + ts-api-utils: ^1.3.0 + peerDependencies: + "@typescript-eslint/parser": ^8.0.0 || ^8.0.0-alpha.0 + eslint: ^8.57.0 || ^9.0.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 5cfc337a957b1c1a868f0f05ed278d4b631aab3aad037c1ca52f458973dee53c2f79db5cb3ac0278d3a4d2846560335212e347c4b978efd84811d6c910e93975 + languageName: node + linkType: hard + "@typescript-eslint/eslint-plugin@npm:^4.30.0": version: 4.33.0 resolution: "@typescript-eslint/eslint-plugin@npm:4.33.0" @@ -12150,6 +12278,24 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/parser@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/parser@npm:8.11.0" + dependencies: + "@typescript-eslint/scope-manager": 8.11.0 + "@typescript-eslint/types": 8.11.0 + "@typescript-eslint/typescript-estree": 8.11.0 + "@typescript-eslint/visitor-keys": 8.11.0 + debug: ^4.3.4 + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: b7664933df72e150289889e16f10f042d427d8334786ce33fa2e0d2fd2fbf31a52c6e88f9b6b9a864f4e78c7b60cd52c034886eb1fa82893d69434bcd4f7e173 + languageName: node + linkType: hard + "@typescript-eslint/parser@npm:^4.30.0": version: 4.33.0 resolution: "@typescript-eslint/parser@npm:4.33.0" @@ -12296,6 +12442,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/scope-manager@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/scope-manager@npm:8.11.0" + dependencies: + "@typescript-eslint/types": 8.11.0 + "@typescript-eslint/visitor-keys": 8.11.0 + checksum: f36212ac1df6a2ed0953beda6bf66e57fd56fcc1c4b4d21149f3451ae621f63aa7ccb92aa1281615250264fdd22e56a163a5d11c5c772c857741ac0e25533325 + languageName: node + linkType: hard + "@typescript-eslint/scope-manager@npm:8.6.0": version: 8.6.0 resolution: "@typescript-eslint/scope-manager@npm:8.6.0" @@ -12373,6 +12529,21 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/type-utils@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/type-utils@npm:8.11.0" + dependencies: + "@typescript-eslint/typescript-estree": 8.11.0 + "@typescript-eslint/utils": 8.11.0 + debug: ^4.3.4 + ts-api-utils: ^1.3.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 74704ee811de343ea2d349a16eec53b6cc8f2b5720510bf327e10667304c48410af78b9ec7aee5d43924a3f6c268cc2cddb7a0606f20c62391b0d7045d8b6264 + languageName: node + linkType: hard + "@typescript-eslint/type-utils@npm:8.6.0": version: 8.6.0 resolution: "@typescript-eslint/type-utils@npm:8.6.0" @@ -12430,6 +12601,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/types@npm:8.11.0" + checksum: 2958f3b5b30d3a876aad79df15662e6c23fe3d0c7750c473f27adc725b2a20f303e914db785c64200bc4092c3489648407792e2bd89eccf3f7aaa4495be81681 + languageName: node + linkType: hard + "@typescript-eslint/types@npm:8.6.0": version: 8.6.0 resolution: "@typescript-eslint/types@npm:8.6.0" @@ -12546,6 +12724,25 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/typescript-estree@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/typescript-estree@npm:8.11.0" + dependencies: + "@typescript-eslint/types": 8.11.0 + "@typescript-eslint/visitor-keys": 8.11.0 + debug: ^4.3.4 + fast-glob: ^3.3.2 + is-glob: ^4.0.3 + minimatch: ^9.0.4 + semver: ^7.6.0 + ts-api-utils: ^1.3.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 03ae4740d4ff19ebc3ea68ac3be1a0265b4abe6348fdc48123e20d6f9206baaa70209e65c9fa4a91930da7d3952c55099a307014284c9b596b12f72bce741817 + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:8.6.0": version: 8.6.0 resolution: "@typescript-eslint/typescript-estree@npm:8.6.0" @@ -12634,6 +12831,20 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/utils@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/utils@npm:8.11.0" + dependencies: + "@eslint-community/eslint-utils": ^4.4.0 + "@typescript-eslint/scope-manager": 8.11.0 + "@typescript-eslint/types": 8.11.0 + "@typescript-eslint/typescript-estree": 8.11.0 + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + checksum: 0a6286fb6c6aaf497bcd5657e4f8167f29c32bb913e4feab3822c504f537ac30975d626dff442cc691e040384ad197313b5685d79296fc8a42ed6c827dcb52fc + languageName: node + linkType: hard + "@typescript-eslint/utils@npm:8.6.0, @typescript-eslint/utils@npm:^6.0.0 || ^7.0.0 || ^8.0.0": version: 8.6.0 resolution: "@typescript-eslint/utils@npm:8.6.0" @@ -12726,6 +12937,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/visitor-keys@npm:8.11.0": + version: 8.11.0 + resolution: "@typescript-eslint/visitor-keys@npm:8.11.0" + dependencies: + "@typescript-eslint/types": 8.11.0 + eslint-visitor-keys: ^3.4.3 + checksum: 29057642bf63994646bd8c5b4baa704ae8b1ff094daa6254a6a92e9fbd252086e219b2b7e8050a131da58cd16cc4dee20bb9fc142bc0d3f22f92af2b59b5444e + languageName: node + linkType: hard + "@typescript-eslint/visitor-keys@npm:8.6.0": version: 8.6.0 resolution: "@typescript-eslint/visitor-keys@npm:8.6.0" @@ -14067,6 +14288,15 @@ __metadata: languageName: node linkType: hard +"acorn@npm:^8.12.0": + version: 8.13.0 + resolution: "acorn@npm:8.13.0" + bin: + acorn: bin/acorn + checksum: f1541f05eb5d6ff67990d1927290809b1ebb663ac96d9c7057c935cf29c5bcaba6d39f37bd007f4bb814f162f142b0f2b2dd4b14128b8fcfaf9f0508a6f05f1c + languageName: node + linkType: hard + "acorn@npm:^8.2.4, acorn@npm:^8.4.1, acorn@npm:^8.7.0": version: 8.7.0 resolution: "acorn@npm:8.7.0" @@ -14695,6 +14925,16 @@ __metadata: languageName: node linkType: hard +"array-buffer-byte-length@npm:^1.0.1": + version: 1.0.1 + resolution: "array-buffer-byte-length@npm:1.0.1" + dependencies: + call-bind: ^1.0.5 + is-array-buffer: ^3.0.4 + checksum: 53524e08f40867f6a9f35318fafe467c32e45e9c682ba67b11943e167344d2febc0f6977a17e699b05699e805c3e8f073d876f8bbf1b559ed494ad2cd0fae09e + languageName: node + linkType: hard + "array-equal@npm:^1.0.0": version: 1.0.0 resolution: "array-equal@npm:1.0.0" @@ -14749,6 +14989,20 @@ __metadata: languageName: node linkType: hard +"array-includes@npm:^3.1.8": + version: 3.1.8 + resolution: "array-includes@npm:3.1.8" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.2 + es-object-atoms: ^1.0.0 + get-intrinsic: ^1.2.4 + is-string: ^1.0.7 + checksum: eb39ba5530f64e4d8acab39297c11c1c5be2a4ea188ab2b34aba5fb7224d918f77717a9d57a3e2900caaa8440e59431bdaf5c974d5212ef65d97f132e38e2d91 + languageName: node + linkType: hard + "array-union@npm:^1.0.1, array-union@npm:^1.0.2": version: 1.0.2 resolution: "array-union@npm:1.0.2" @@ -14786,6 +15040,20 @@ __metadata: languageName: node linkType: hard +"array.prototype.findlastindex@npm:^1.2.5": + version: 1.2.5 + resolution: "array.prototype.findlastindex@npm:1.2.5" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.2 + es-errors: ^1.3.0 + es-object-atoms: ^1.0.0 + es-shim-unscopables: ^1.0.2 + checksum: 2c81cff2a75deb95bf1ed89b6f5f2bfbfb882211e3b7cc59c3d6b87df774cd9d6b36949a8ae39ac476e092c1d4a4905f5ee11a86a456abb10f35f8211ae4e710 + languageName: node + linkType: hard + "array.prototype.flat@npm:^1.2.1, array.prototype.flat@npm:^1.2.5": version: 1.2.5 resolution: "array.prototype.flat@npm:1.2.5" @@ -14809,6 +15077,18 @@ __metadata: languageName: node linkType: hard +"array.prototype.flat@npm:^1.3.2": + version: 1.3.2 + resolution: "array.prototype.flat@npm:1.3.2" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.2.0 + es-abstract: ^1.22.1 + es-shim-unscopables: ^1.0.0 + checksum: 5d6b4bf102065fb3f43764bfff6feb3295d372ce89591e6005df3d0ce388527a9f03c909af6f2a973969a4d178ab232ffc9236654149173e0e187ec3a1a6b87b + languageName: node + linkType: hard + "array.prototype.flatmap@npm:^1.2.1": version: 1.2.5 resolution: "array.prototype.flatmap@npm:1.2.5" @@ -14832,6 +15112,18 @@ __metadata: languageName: node linkType: hard +"array.prototype.flatmap@npm:^1.3.2": + version: 1.3.2 + resolution: "array.prototype.flatmap@npm:1.3.2" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.2.0 + es-abstract: ^1.22.1 + es-shim-unscopables: ^1.0.0 + checksum: ce09fe21dc0bcd4f30271f8144083aa8c13d4639074d6c8dc82054b847c7fc9a0c97f857491f4da19d4003e507172a78f4bcd12903098adac8b9cd374f734be3 + languageName: node + linkType: hard + "array.prototype.map@npm:^1.0.4": version: 1.0.4 resolution: "array.prototype.map@npm:1.0.4" @@ -14873,6 +15165,22 @@ __metadata: languageName: node linkType: hard +"arraybuffer.prototype.slice@npm:^1.0.3": + version: 1.0.3 + resolution: "arraybuffer.prototype.slice@npm:1.0.3" + dependencies: + array-buffer-byte-length: ^1.0.1 + call-bind: ^1.0.5 + define-properties: ^1.2.1 + es-abstract: ^1.22.3 + es-errors: ^1.2.1 + get-intrinsic: ^1.2.3 + is-array-buffer: ^3.0.4 + is-shared-array-buffer: ^1.0.2 + checksum: 352259cba534dcdd969c92ab002efd2ba5025b2e3b9bead3973150edbdf0696c629d7f4b3f061c5931511e8207bdc2306da614703c820b45dabce39e3daf7e3e + languageName: node + linkType: hard + "arrify@npm:^1.0.1": version: 1.0.1 resolution: "arrify@npm:1.0.1" @@ -15128,6 +15436,15 @@ __metadata: languageName: node linkType: hard +"available-typed-arrays@npm:^1.0.7": + version: 1.0.7 + resolution: "available-typed-arrays@npm:1.0.7" + dependencies: + possible-typed-array-names: ^1.0.0 + checksum: 1aa3ffbfe6578276996de660848b6e95669d9a95ad149e3dd0c0cda77db6ee1dbd9d1dd723b65b6d277b882dd0c4b91a654ae9d3cf9e1254b7e93e4908d78fd3 + languageName: node + linkType: hard + "aws-sign2@npm:~0.7.0": version: 0.7.0 resolution: "aws-sign2@npm:0.7.0" @@ -16369,20 +16686,6 @@ __metadata: languageName: node linkType: hard -"browserslist@npm:^4.23.3": - version: 4.24.0 - resolution: "browserslist@npm:4.24.0" - dependencies: - caniuse-lite: ^1.0.30001663 - electron-to-chromium: ^1.5.28 - node-releases: ^2.0.18 - update-browserslist-db: ^1.1.0 - bin: - browserslist: cli.js - checksum: de200d3eb8d6ed819dad99719099a28fb6ebeb88016a5ac42fbdc11607e910c236a84ca1b0bbf232477d4b88ab64e8ab6aa67557cdd40a73ca9c2834f92ccce0 - languageName: node - linkType: hard - "bs-logger@npm:0.x, bs-logger@npm:^0.2.6": version: 0.2.6 resolution: "bs-logger@npm:0.2.6" @@ -16488,13 +16791,6 @@ __metadata: languageName: node linkType: hard -"builtin-modules@npm:^3.3.0": - version: 3.3.0 - resolution: "builtin-modules@npm:3.3.0" - checksum: db021755d7ed8be048f25668fe2117620861ef6703ea2c65ed2779c9e3636d5c3b82325bd912244293959ff3ae303afa3471f6a15bf5060c103e4cc3a839749d - languageName: node - linkType: hard - "builtin-status-codes@npm:^3.0.0": version: 3.0.0 resolution: "builtin-status-codes@npm:3.0.0" @@ -16761,6 +17057,19 @@ __metadata: languageName: node linkType: hard +"call-bind@npm:^1.0.6, call-bind@npm:^1.0.7": + version: 1.0.7 + resolution: "call-bind@npm:1.0.7" + dependencies: + es-define-property: ^1.0.0 + es-errors: ^1.3.0 + function-bind: ^1.1.2 + get-intrinsic: ^1.2.4 + set-function-length: ^1.2.1 + checksum: 295c0c62b90dd6522e6db3b0ab1ce26bdf9e7404215bda13cfee25b626b5ff1a7761324d58d38b1ef1607fc65aca2d06e44d2e18d0dfc6c14b465b00d8660029 + languageName: node + linkType: hard + "call-me-maybe@npm:^1.0.1": version: 1.0.1 resolution: "call-me-maybe@npm:1.0.1" @@ -16871,13 +17180,6 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001663": - version: 1.0.30001669 - resolution: "caniuse-lite@npm:1.0.30001669" - checksum: 8ed0c69d0c6aa3b1cbc5ba4e5f5330943e7b7165e257f6955b8b73f043d07ad922265261f2b54d9bbaf02886bbdba5e6f5b16662310a13f91f17035af3212de1 - languageName: node - linkType: hard - "capture-exit@npm:^2.0.0": version: 2.0.0 resolution: "capture-exit@npm:2.0.0" @@ -17159,13 +17461,6 @@ __metadata: languageName: node linkType: hard -"ci-info@npm:^4.0.0": - version: 4.0.0 - resolution: "ci-info@npm:4.0.0" - checksum: 122fe41c5eb8d0b5fa0ab6fd674c5ddcf2dc59766528b062a0144ff0d913cfb210ef925ec52110e7c2a7f4e603d5f0e8b91cfe68867e196e9212fa0b94d0a08a - languageName: node - linkType: hard - "cidr-regex@npm:^3.1.1": version: 3.1.1 resolution: "cidr-regex@npm:3.1.1" @@ -18096,15 +18391,6 @@ __metadata: languageName: node linkType: hard -"core-js-compat@npm:^3.38.1": - version: 3.38.1 - resolution: "core-js-compat@npm:3.38.1" - dependencies: - browserslist: ^4.23.3 - checksum: a0a5673bcd59f588f0cd0b59cdacd4712b82909738a87406d334dd412eb3d273ae72b275bdd8e8fef63fca9ef12b42ed651be139c7c44c8a1acb423c8906992e - languageName: node - linkType: hard - "core-js@npm:^2.4.0, core-js@npm:^2.5.0": version: 2.6.12 resolution: "core-js@npm:2.6.12" @@ -19309,6 +19595,39 @@ __metadata: languageName: node linkType: hard +"data-view-buffer@npm:^1.0.1": + version: 1.0.1 + resolution: "data-view-buffer@npm:1.0.1" + dependencies: + call-bind: ^1.0.6 + es-errors: ^1.3.0 + is-data-view: ^1.0.1 + checksum: ce24348f3c6231223b216da92e7e6a57a12b4af81a23f27eff8feabdf06acfb16c00639c8b705ca4d167f761cfc756e27e5f065d0a1f840c10b907fdaf8b988c + languageName: node + linkType: hard + +"data-view-byte-length@npm:^1.0.1": + version: 1.0.1 + resolution: "data-view-byte-length@npm:1.0.1" + dependencies: + call-bind: ^1.0.7 + es-errors: ^1.3.0 + is-data-view: ^1.0.1 + checksum: dbb3200edcb7c1ef0d68979834f81d64fd8cab2f7691b3a4c6b97e67f22182f3ec2c8602efd7b76997b55af6ff8bce485829c1feda4fa2165a6b71fb7baa4269 + languageName: node + linkType: hard + +"data-view-byte-offset@npm:^1.0.0": + version: 1.0.0 + resolution: "data-view-byte-offset@npm:1.0.0" + dependencies: + call-bind: ^1.0.6 + es-errors: ^1.3.0 + is-data-view: ^1.0.1 + checksum: 7f0bf8720b7414ca719eedf1846aeec392f2054d7af707c5dc9a753cc77eb8625f067fa901e0b5127e831f9da9056138d894b9c2be79c27a21f6db5824f009c2 + languageName: node + linkType: hard + "dateformat@npm:^3.0.0": version: 3.0.3 resolution: "dateformat@npm:3.0.3" @@ -19640,6 +19959,17 @@ __metadata: languageName: node linkType: hard +"define-data-property@npm:^1.1.4": + version: 1.1.4 + resolution: "define-data-property@npm:1.1.4" + dependencies: + es-define-property: ^1.0.0 + es-errors: ^1.3.0 + gopd: ^1.0.1 + checksum: 8068ee6cab694d409ac25936eb861eea704b7763f7f342adbdfe337fc27c78d7ae0eff2364b2917b58c508d723c7a074326d068eef2e45c4edcd85cf94d0313b + languageName: node + linkType: hard + "define-lazy-prop@npm:^2.0.0": version: 2.0.0 resolution: "define-lazy-prop@npm:2.0.0" @@ -20463,13 +20793,6 @@ __metadata: languageName: node linkType: hard -"electron-to-chromium@npm:^1.5.28": - version: 1.5.41 - resolution: "electron-to-chromium@npm:1.5.41" - checksum: 942cc53beabeb0647598d432155e2c21bed0de3dfd46576112aeed4157ea59543875c8a99038c5b05e8843fb3b91f14278ed4ea2bf4943845b26456ec20d2c9b - languageName: node - linkType: hard - "elkjs@npm:^0.8.2": version: 0.8.2 resolution: "elkjs@npm:0.8.2" @@ -20851,6 +21174,60 @@ __metadata: languageName: node linkType: hard +"es-abstract@npm:^1.22.3, es-abstract@npm:^1.23.0, es-abstract@npm:^1.23.2": + version: 1.23.3 + resolution: "es-abstract@npm:1.23.3" + dependencies: + array-buffer-byte-length: ^1.0.1 + arraybuffer.prototype.slice: ^1.0.3 + available-typed-arrays: ^1.0.7 + call-bind: ^1.0.7 + data-view-buffer: ^1.0.1 + data-view-byte-length: ^1.0.1 + data-view-byte-offset: ^1.0.0 + es-define-property: ^1.0.0 + es-errors: ^1.3.0 + es-object-atoms: ^1.0.0 + es-set-tostringtag: ^2.0.3 + es-to-primitive: ^1.2.1 + function.prototype.name: ^1.1.6 + get-intrinsic: ^1.2.4 + get-symbol-description: ^1.0.2 + globalthis: ^1.0.3 + gopd: ^1.0.1 + has-property-descriptors: ^1.0.2 + has-proto: ^1.0.3 + has-symbols: ^1.0.3 + hasown: ^2.0.2 + internal-slot: ^1.0.7 + is-array-buffer: ^3.0.4 + is-callable: ^1.2.7 + is-data-view: ^1.0.1 + is-negative-zero: ^2.0.3 + is-regex: ^1.1.4 + is-shared-array-buffer: ^1.0.3 + is-string: ^1.0.7 + is-typed-array: ^1.1.13 + is-weakref: ^1.0.2 + object-inspect: ^1.13.1 + object-keys: ^1.1.1 + object.assign: ^4.1.5 + regexp.prototype.flags: ^1.5.2 + safe-array-concat: ^1.1.2 + safe-regex-test: ^1.0.3 + string.prototype.trim: ^1.2.9 + string.prototype.trimend: ^1.0.8 + string.prototype.trimstart: ^1.0.8 + typed-array-buffer: ^1.0.2 + typed-array-byte-length: ^1.0.1 + typed-array-byte-offset: ^1.0.2 + typed-array-length: ^1.0.6 + unbox-primitive: ^1.0.2 + which-typed-array: ^1.1.15 + checksum: f840cf161224252512f9527306b57117192696571e07920f777cb893454e32999206198b4f075516112af6459daca282826d1735c450528470356d09eff3a9ae + languageName: node + linkType: hard + "es-array-method-boxes-properly@npm:^1.0.0": version: 1.0.0 resolution: "es-array-method-boxes-properly@npm:1.0.0" @@ -20858,6 +21235,22 @@ __metadata: languageName: node linkType: hard +"es-define-property@npm:^1.0.0": + version: 1.0.0 + resolution: "es-define-property@npm:1.0.0" + dependencies: + get-intrinsic: ^1.2.4 + checksum: f66ece0a887b6dca71848fa71f70461357c0e4e7249696f81bad0a1f347eed7b31262af4a29f5d726dc026426f085483b6b90301855e647aa8e21936f07293c6 + languageName: node + linkType: hard + +"es-errors@npm:^1.2.1, es-errors@npm:^1.3.0": + version: 1.3.0 + resolution: "es-errors@npm:1.3.0" + checksum: ec1414527a0ccacd7f15f4a3bc66e215f04f595ba23ca75cdae0927af099b5ec865f9f4d33e9d7e86f512f252876ac77d4281a7871531a50678132429b1271b5 + languageName: node + linkType: hard + "es-get-iterator@npm:^1.0.2": version: 1.1.2 resolution: "es-get-iterator@npm:1.1.2" @@ -20927,6 +21320,15 @@ __metadata: languageName: node linkType: hard +"es-object-atoms@npm:^1.0.0": + version: 1.0.0 + resolution: "es-object-atoms@npm:1.0.0" + dependencies: + es-errors: ^1.3.0 + checksum: 26f0ff78ab93b63394e8403c353842b2272836968de4eafe97656adfb8a7c84b9099bf0fe96ed58f4a4cddc860f6e34c77f91649a58a5daa4a9c40b902744e3c + languageName: node + linkType: hard + "es-set-tostringtag@npm:^2.0.1": version: 2.0.1 resolution: "es-set-tostringtag@npm:2.0.1" @@ -20938,6 +21340,17 @@ __metadata: languageName: node linkType: hard +"es-set-tostringtag@npm:^2.0.3": + version: 2.0.3 + resolution: "es-set-tostringtag@npm:2.0.3" + dependencies: + get-intrinsic: ^1.2.4 + has-tostringtag: ^1.0.2 + hasown: ^2.0.1 + checksum: 7227fa48a41c0ce83e0377b11130d324ac797390688135b8da5c28994c0165be8b252e15cd1de41e1325e5a5412511586960213e88f9ab4a5e7d028895db5129 + languageName: node + linkType: hard + "es-shim-unscopables@npm:^1.0.0": version: 1.0.0 resolution: "es-shim-unscopables@npm:1.0.0" @@ -20947,6 +21360,15 @@ __metadata: languageName: node linkType: hard +"es-shim-unscopables@npm:^1.0.2": + version: 1.0.2 + resolution: "es-shim-unscopables@npm:1.0.2" + dependencies: + hasown: ^2.0.0 + checksum: 432bd527c62065da09ed1d37a3f8e623c423683285e6188108286f4a1e8e164a5bcbfbc0051557c7d14633cd2a41ce24c7048e6bbb66a985413fd32f1be72626 + languageName: node + linkType: hard + "es-to-primitive@npm:^1.2.1": version: 1.2.1 resolution: "es-to-primitive@npm:1.2.1" @@ -21384,13 +21806,6 @@ __metadata: languageName: node linkType: hard -"escalade@npm:^3.2.0": - version: 3.2.0 - resolution: "escalade@npm:3.2.0" - checksum: 47b029c83de01b0d17ad99ed766347b974b0d628e848de404018f3abee728e987da0d2d370ad4574aa3d5b5bfc368754fd085d69a30f8e75903486ec4b5b709e - languageName: node - linkType: hard - "escape-html@npm:~1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" @@ -21529,6 +21944,17 @@ __metadata: languageName: node linkType: hard +"eslint-config-prettier@npm:^9": + version: 9.1.0 + resolution: "eslint-config-prettier@npm:9.1.0" + peerDependencies: + eslint: ">=7.0.0" + bin: + eslint-config-prettier: bin/cli.js + checksum: 9229b768c879f500ee54ca05925f31b0c0bafff3d9f5521f98ff05127356de78c81deb9365c86a5ec4efa990cb72b74df8612ae15965b14136044c73e1f6a907 + languageName: node + linkType: hard + "eslint-formatter-pretty@npm:^4.1.0": version: 4.1.0 resolution: "eslint-formatter-pretty@npm:4.1.0" @@ -21578,6 +22004,29 @@ __metadata: languageName: node linkType: hard +"eslint-import-resolver-node@npm:^0.3.9": + version: 0.3.9 + resolution: "eslint-import-resolver-node@npm:0.3.9" + dependencies: + debug: ^3.2.7 + is-core-module: ^2.13.0 + resolve: ^1.22.4 + checksum: 439b91271236b452d478d0522a44482e8c8540bf9df9bd744062ebb89ab45727a3acd03366a6ba2bdbcde8f9f718bab7fe8db64688aca75acf37e04eafd25e22 + languageName: node + linkType: hard + +"eslint-module-utils@npm:^2.12.0": + version: 2.12.0 + resolution: "eslint-module-utils@npm:2.12.0" + dependencies: + debug: ^3.2.7 + peerDependenciesMeta: + eslint: + optional: true + checksum: be3ac52e0971c6f46daeb1a7e760e45c7c45f820c8cc211799f85f10f04ccbf7afc17039165d56cb2da7f7ca9cec2b3a777013cddf0b976784b37eb9efa24180 + languageName: node + linkType: hard + "eslint-module-utils@npm:^2.7.2": version: 2.7.3 resolution: "eslint-module-utils@npm:2.7.3" @@ -21672,6 +22121,35 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-import@npm:^2.31.0": + version: 2.31.0 + resolution: "eslint-plugin-import@npm:2.31.0" + dependencies: + "@rtsao/scc": ^1.1.0 + array-includes: ^3.1.8 + array.prototype.findlastindex: ^1.2.5 + array.prototype.flat: ^1.3.2 + array.prototype.flatmap: ^1.3.2 + debug: ^3.2.7 + doctrine: ^2.1.0 + eslint-import-resolver-node: ^0.3.9 + eslint-module-utils: ^2.12.0 + hasown: ^2.0.2 + is-core-module: ^2.15.1 + is-glob: ^4.0.3 + minimatch: ^3.1.2 + object.fromentries: ^2.0.8 + object.groupby: ^1.0.3 + object.values: ^1.2.0 + semver: ^6.3.1 + string.prototype.trimend: ^1.0.8 + tsconfig-paths: ^3.15.0 + peerDependencies: + eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9 + checksum: b1d2ac268b3582ff1af2a72a2c476eae4d250c100f2e335b6e102036e4a35efa530b80ec578dfc36761fabb34a635b9bf5ab071abe9d4404a4bb054fdf22d415 + languageName: node + linkType: hard + "eslint-plugin-jest@npm:^25.3.0": version: 25.7.0 resolution: "eslint-plugin-jest@npm:25.7.0" @@ -21764,6 +22242,26 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-prettier@npm:^5.2.1": + version: 5.2.1 + resolution: "eslint-plugin-prettier@npm:5.2.1" + dependencies: + prettier-linter-helpers: ^1.0.0 + synckit: ^0.9.1 + peerDependencies: + "@types/eslint": ">=8.0.0" + eslint: ">=8.0.0" + eslint-config-prettier: "*" + prettier: ">=3.0.0" + peerDependenciesMeta: + "@types/eslint": + optional: true + eslint-config-prettier: + optional: true + checksum: 812f4d1596dcd3a55963212dfbd818a4b38f880741aac75f6869aa740dc5d934060674d3b85d10ff9fec424defa61967dbdef26b8a893a92c9b51880264ed0d9 + languageName: node + linkType: hard + "eslint-plugin-promise@npm:^6.0.1": version: 6.0.1 resolution: "eslint-plugin-promise@npm:6.0.1" @@ -21872,32 +22370,6 @@ __metadata: languageName: node linkType: hard -"eslint-plugin-unicorn@npm:latest": - version: 56.0.0 - resolution: "eslint-plugin-unicorn@npm:56.0.0" - dependencies: - "@babel/helper-validator-identifier": ^7.24.7 - "@eslint-community/eslint-utils": ^4.4.0 - ci-info: ^4.0.0 - clean-regexp: ^1.0.0 - core-js-compat: ^3.38.1 - esquery: ^1.6.0 - globals: ^15.9.0 - indent-string: ^4.0.0 - is-builtin-module: ^3.2.1 - jsesc: ^3.0.2 - pluralize: ^8.0.0 - read-pkg-up: ^7.0.1 - regexp-tree: ^0.1.27 - regjsparser: ^0.10.0 - semver: ^7.6.3 - strip-indent: ^3.0.0 - peerDependencies: - eslint: ">=8.56.0" - checksum: 4bc05fac0b49523de08452ebf0238d2e7f7b5790c234b22b9d74035efa656f01d79f7adcff8e5b075e9813e3cae9c36ba8e1956c1b12449c64c7af9c6a36726d - languageName: node - linkType: hard - "eslint-plugin-vue@npm:^9.9.0": version: 9.9.0 resolution: "eslint-plugin-vue@npm:9.9.0" @@ -21975,6 +22447,16 @@ __metadata: languageName: node linkType: hard +"eslint-scope@npm:^8.1.0": + version: 8.1.0 + resolution: "eslint-scope@npm:8.1.0" + dependencies: + esrecurse: ^4.3.0 + estraverse: ^5.2.0 + checksum: 3ce9392ec74f35f84eddad7755941cb5f7e3a1bc53cf902e95e541384ef78b604bec1489933f37c663e4686b36c31533d06cf0842be9a729a680c489efce7acb + languageName: node + linkType: hard + "eslint-template-visitor@npm:^2.3.2": version: 2.3.2 resolution: "eslint-template-visitor@npm:2.3.2" @@ -22038,6 +22520,13 @@ __metadata: languageName: node linkType: hard +"eslint-visitor-keys@npm:^4.1.0": + version: 4.1.0 + resolution: "eslint-visitor-keys@npm:4.1.0" + checksum: b5d53725df14a6a225fd74d5e687f5f0547b0aaa3e1963ab6f4acb8dc80f99ad0bec11148e14b4a67de024dde7b4449e7e4c0b1524de605955dee7eefcdd7824 + languageName: node + linkType: hard + "eslint@npm:^7.25.0, eslint@npm:^7.28.0, eslint@npm:^7.32.0": version: 7.32.0 resolution: "eslint@npm:7.32.0" @@ -22181,6 +22670,67 @@ __metadata: languageName: node linkType: hard +"eslint@npm:^9": + version: 9.13.0 + resolution: "eslint@npm:9.13.0" + dependencies: + "@eslint-community/eslint-utils": ^4.2.0 + "@eslint-community/regexpp": ^4.11.0 + "@eslint/config-array": ^0.18.0 + "@eslint/core": ^0.7.0 + "@eslint/eslintrc": ^3.1.0 + "@eslint/js": 9.13.0 + "@eslint/plugin-kit": ^0.2.0 + "@humanfs/node": ^0.16.5 + "@humanwhocodes/module-importer": ^1.0.1 + "@humanwhocodes/retry": ^0.3.1 + "@types/estree": ^1.0.6 + "@types/json-schema": ^7.0.15 + ajv: ^6.12.4 + chalk: ^4.0.0 + cross-spawn: ^7.0.2 + debug: ^4.3.2 + escape-string-regexp: ^4.0.0 + eslint-scope: ^8.1.0 + eslint-visitor-keys: ^4.1.0 + espree: ^10.2.0 + esquery: ^1.5.0 + esutils: ^2.0.2 + fast-deep-equal: ^3.1.3 + file-entry-cache: ^8.0.0 + find-up: ^5.0.0 + glob-parent: ^6.0.2 + ignore: ^5.2.0 + imurmurhash: ^0.1.4 + is-glob: ^4.0.0 + json-stable-stringify-without-jsonify: ^1.0.1 + lodash.merge: ^4.6.2 + minimatch: ^3.1.2 + natural-compare: ^1.4.0 + optionator: ^0.9.3 + text-table: ^0.2.0 + peerDependencies: + jiti: "*" + peerDependenciesMeta: + jiti: + optional: true + bin: + eslint: bin/eslint.js + checksum: 99e878d6883864b8361bfaf2a2304f1e133347ac19976c79e1430623cd311cb38253bbd122100788082eded947693cce5c7e67dfd2b5173e6f05edb92dcb2206 + languageName: node + linkType: hard + +"espree@npm:^10.0.1, espree@npm:^10.2.0": + version: 10.2.0 + resolution: "espree@npm:10.2.0" + dependencies: + acorn: ^8.12.0 + acorn-jsx: ^5.3.2 + eslint-visitor-keys: ^4.1.0 + checksum: 16ee75c2f6029622a70a675ad8989fffc6f7199265d07af516a11e4adc9eb2d03866fceff33f1a081c42621df79871e508f8fc8fe5855eac2de925b58196711b + languageName: node + linkType: hard + "espree@npm:^7.3.0, espree@npm:^7.3.1": version: 7.3.1 resolution: "espree@npm:7.3.1" @@ -22242,7 +22792,7 @@ __metadata: languageName: node linkType: hard -"esquery@npm:^1.6.0": +"esquery@npm:^1.5.0": version: 1.6.0 resolution: "esquery@npm:1.6.0" dependencies: @@ -22972,6 +23522,15 @@ __metadata: languageName: node linkType: hard +"file-entry-cache@npm:^8.0.0": + version: 8.0.0 + resolution: "file-entry-cache@npm:8.0.0" + dependencies: + flat-cache: ^4.0.0 + checksum: f67802d3334809048c69b3d458f672e1b6d26daefda701761c81f203b80149c35dea04d78ea4238969dd617678e530876722a0634c43031a0957f10cc3ed190f + languageName: node + linkType: hard + "file-loader@npm:^4.2.0": version: 4.3.0 resolution: "file-loader@npm:4.3.0" @@ -23218,6 +23777,16 @@ __metadata: languageName: node linkType: hard +"flat-cache@npm:^4.0.0": + version: 4.0.1 + resolution: "flat-cache@npm:4.0.1" + dependencies: + flatted: ^3.2.9 + keyv: ^4.5.4 + checksum: 899fc86bf6df093547d76e7bfaeb900824b869d7d457d02e9b8aae24836f0a99fbad79328cfd6415ee8908f180699bf259dc7614f793447cb14f707caf5996f6 + languageName: node + linkType: hard + "flatted@npm:^3.1.0": version: 3.2.5 resolution: "flatted@npm:3.2.5" @@ -23225,6 +23794,13 @@ __metadata: languageName: node linkType: hard +"flatted@npm:^3.2.9": + version: 3.3.1 + resolution: "flatted@npm:3.3.1" + checksum: 85ae7181650bb728c221e7644cbc9f4bf28bc556f2fc89bb21266962bdf0ce1029cc7acc44bb646cd469d9baac7c317f64e841c4c4c00516afa97320cdac7f94 + languageName: node + linkType: hard + "flow-parser@npm:0.*": version: 0.171.0 resolution: "flow-parser@npm:0.171.0" @@ -23872,6 +24448,19 @@ __metadata: languageName: node linkType: hard +"get-intrinsic@npm:^1.2.3, get-intrinsic@npm:^1.2.4": + version: 1.2.4 + resolution: "get-intrinsic@npm:1.2.4" + dependencies: + es-errors: ^1.3.0 + function-bind: ^1.1.2 + has-proto: ^1.0.1 + has-symbols: ^1.0.3 + hasown: ^2.0.0 + checksum: 414e3cdf2c203d1b9d7d33111df746a4512a1aa622770b361dadddf8ed0b5aeb26c560f49ca077e24bfafb0acb55ca908d1f709216ccba33ffc548ec8a79a951 + languageName: node + linkType: hard + "get-nonce@npm:^1.0.0": version: 1.0.1 resolution: "get-nonce@npm:1.0.1" @@ -23949,6 +24538,17 @@ __metadata: languageName: node linkType: hard +"get-symbol-description@npm:^1.0.2": + version: 1.0.2 + resolution: "get-symbol-description@npm:1.0.2" + dependencies: + call-bind: ^1.0.5 + es-errors: ^1.3.0 + get-intrinsic: ^1.2.4 + checksum: e1cb53bc211f9dbe9691a4f97a46837a553c4e7caadd0488dc24ac694db8a390b93edd412b48dcdd0b4bbb4c595de1709effc75fc87c0839deedc6968f5bd973 + languageName: node + linkType: hard + "get-value@npm:^2.0.3, get-value@npm:^2.0.6": version: 2.0.6 resolution: "get-value@npm:2.0.6" @@ -24300,7 +24900,14 @@ __metadata: languageName: node linkType: hard -"globals@npm:^15.9.0": +"globals@npm:^14.0.0": + version: 14.0.0 + resolution: "globals@npm:14.0.0" + checksum: 534b8216736a5425737f59f6e6a5c7f386254560c9f41d24a9227d60ee3ad4a9e82c5b85def0e212e9d92162f83a92544be4c7fd4c902cb913736c10e08237ac + languageName: node + linkType: hard + +"globals@npm:^15.11.0": version: 15.11.0 resolution: "globals@npm:15.11.0" checksum: ef32d5ef987f3d4b47fc2e389a0b235f6a46f605160c4e405722fd7b576106ca407cb867e66fd1e0fc43b631800e2e2e71847f37691026d813f96f40339da702 @@ -24612,6 +25219,15 @@ __metadata: languageName: node linkType: hard +"has-property-descriptors@npm:^1.0.2": + version: 1.0.2 + resolution: "has-property-descriptors@npm:1.0.2" + dependencies: + es-define-property: ^1.0.0 + checksum: fcbb246ea2838058be39887935231c6d5788babed499d0e9d0cc5737494c48aba4fe17ba1449e0d0fbbb1e36175442faa37f9c427ae357d6ccb1d895fbcd3de3 + languageName: node + linkType: hard + "has-proto@npm:^1.0.1": version: 1.0.1 resolution: "has-proto@npm:1.0.1" @@ -24619,6 +25235,13 @@ __metadata: languageName: node linkType: hard +"has-proto@npm:^1.0.3": + version: 1.0.3 + resolution: "has-proto@npm:1.0.3" + checksum: fe7c3d50b33f50f3933a04413ed1f69441d21d2d2944f81036276d30635cad9279f6b43bc8f32036c31ebdfcf6e731150f46c1907ad90c669ffe9b066c3ba5c4 + languageName: node + linkType: hard + "has-symbols@npm:^1.0.1, has-symbols@npm:^1.0.2": version: 1.0.2 resolution: "has-symbols@npm:1.0.2" @@ -24642,6 +25265,15 @@ __metadata: languageName: node linkType: hard +"has-tostringtag@npm:^1.0.2": + version: 1.0.2 + resolution: "has-tostringtag@npm:1.0.2" + dependencies: + has-symbols: ^1.0.3 + checksum: 999d60bb753ad714356b2c6c87b7fb74f32463b8426e159397da4bde5bca7e598ab1073f4d8d4deafac297f2eb311484cd177af242776bf05f0d11565680468d + languageName: node + linkType: hard + "has-unicode@npm:^2.0.0, has-unicode@npm:^2.0.1": version: 2.0.1 resolution: "has-unicode@npm:2.0.1" @@ -24741,6 +25373,15 @@ __metadata: languageName: node linkType: hard +"hasown@npm:^2.0.1, hasown@npm:^2.0.2": + version: 2.0.2 + resolution: "hasown@npm:2.0.2" + dependencies: + function-bind: ^1.1.2 + checksum: e8516f776a15149ca6c6ed2ae3110c417a00b62260e222590e54aa367cbcd6ed99122020b37b7fbdf05748df57b265e70095d7bf35a47660587619b15ffb93db + languageName: node + linkType: hard + "he@npm:1.2.x, he@npm:^1.2.0": version: 1.2.0 resolution: "he@npm:1.2.0" @@ -25634,6 +26275,17 @@ __metadata: languageName: node linkType: hard +"internal-slot@npm:^1.0.7": + version: 1.0.7 + resolution: "internal-slot@npm:1.0.7" + dependencies: + es-errors: ^1.3.0 + hasown: ^2.0.0 + side-channel: ^1.0.4 + checksum: cadc5eea5d7d9bc2342e93aae9f31f04c196afebb11bde97448327049f492cd7081e18623ae71388aac9cd237b692ca3a105be9c68ac39c1dec679d7409e33eb + languageName: node + linkType: hard + "internmap@npm:1 - 2": version: 2.0.3 resolution: "internmap@npm:2.0.3" @@ -25804,6 +26456,16 @@ __metadata: languageName: node linkType: hard +"is-array-buffer@npm:^3.0.4": + version: 3.0.4 + resolution: "is-array-buffer@npm:3.0.4" + dependencies: + call-bind: ^1.0.2 + get-intrinsic: ^1.2.1 + checksum: e4e3e6ef0ff2239e75371d221f74bc3c26a03564a22efb39f6bb02609b598917ddeecef4e8c877df2a25888f247a98198959842a5e73236bc7f22cabdf6351a7 + languageName: node + linkType: hard + "is-arrayish@npm:^0.2.1": version: 0.2.1 resolution: "is-arrayish@npm:0.2.1" @@ -25880,15 +26542,6 @@ __metadata: languageName: node linkType: hard -"is-builtin-module@npm:^3.2.1": - version: 3.2.1 - resolution: "is-builtin-module@npm:3.2.1" - dependencies: - builtin-modules: ^3.3.0 - checksum: e8f0ffc19a98240bda9c7ada84d846486365af88d14616e737d280d378695c8c448a621dcafc8332dbf0fcd0a17b0763b845400709963fa9151ddffece90ae88 - languageName: node - linkType: hard - "is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.4": version: 1.2.4 resolution: "is-callable@npm:1.2.4" @@ -25975,6 +26628,15 @@ __metadata: languageName: node linkType: hard +"is-core-module@npm:^2.13.0, is-core-module@npm:^2.15.1": + version: 2.15.1 + resolution: "is-core-module@npm:2.15.1" + dependencies: + hasown: ^2.0.2 + checksum: df134c168115690724b62018c37b2f5bba0d5745fa16960b329c5a00883a8bea6a5632fdb1e3efcce237c201826ba09f93197b7cd95577ea56b0df335be23633 + languageName: node + linkType: hard + "is-core-module@npm:^2.5.0, is-core-module@npm:^2.8.0, is-core-module@npm:^2.8.1": version: 2.8.1 resolution: "is-core-module@npm:2.8.1" @@ -26002,6 +26664,15 @@ __metadata: languageName: node linkType: hard +"is-data-view@npm:^1.0.1": + version: 1.0.1 + resolution: "is-data-view@npm:1.0.1" + dependencies: + is-typed-array: ^1.1.13 + checksum: 4ba4562ac2b2ec005fefe48269d6bd0152785458cd253c746154ffb8a8ab506a29d0cfb3b74af87513843776a88e4981ae25c89457bf640a33748eab1a7216b5 + languageName: node + linkType: hard + "is-date-object@npm:^1.0.1, is-date-object@npm:^1.0.5": version: 1.0.5 resolution: "is-date-object@npm:1.0.5" @@ -26263,6 +26934,13 @@ __metadata: languageName: node linkType: hard +"is-negative-zero@npm:^2.0.3": + version: 2.0.3 + resolution: "is-negative-zero@npm:2.0.3" + checksum: c1e6b23d2070c0539d7b36022d5a94407132411d01aba39ec549af824231f3804b1aea90b5e4e58e807a65d23ceb538ed6e355ce76b267bdd86edb757ffcbdcd + languageName: node + linkType: hard + "is-number-object@npm:^1.0.4": version: 1.0.6 resolution: "is-number-object@npm:1.0.6" @@ -26478,6 +27156,15 @@ __metadata: languageName: node linkType: hard +"is-shared-array-buffer@npm:^1.0.3": + version: 1.0.3 + resolution: "is-shared-array-buffer@npm:1.0.3" + dependencies: + call-bind: ^1.0.7 + checksum: a4fff602c309e64ccaa83b859255a43bb011145a42d3f56f67d9268b55bc7e6d98a5981a1d834186ad3105d6739d21547083fe7259c76c0468483fc538e716d8 + languageName: node + linkType: hard + "is-ssh@npm:^1.0.0, is-ssh@npm:^1.3.0": version: 1.4.0 resolution: "is-ssh@npm:1.4.0" @@ -26550,6 +27237,15 @@ __metadata: languageName: node linkType: hard +"is-typed-array@npm:^1.1.13": + version: 1.1.13 + resolution: "is-typed-array@npm:1.1.13" + dependencies: + which-typed-array: ^1.1.14 + checksum: 150f9ada183a61554c91e1c4290086d2c100b0dff45f60b028519be72a8db964da403c48760723bf5253979b8dffe7b544246e0e5351dcd05c5fdb1dcc1dc0f0 + languageName: node + linkType: hard + "is-typedarray@npm:^1.0.0, is-typedarray@npm:~1.0.0": version: 1.0.0 resolution: "is-typedarray@npm:1.0.0" @@ -29177,15 +29873,6 @@ __metadata: languageName: node linkType: hard -"jsesc@npm:^3.0.2": - version: 3.0.2 - resolution: "jsesc@npm:3.0.2" - bin: - jsesc: bin/jsesc - checksum: a36d3ca40574a974d9c2063bf68c2b6141c20da8f2a36bd3279fc802563f35f0527a6c828801295bdfb2803952cf2cf387786c2c90ed564f88d5782475abfe3c - languageName: node - linkType: hard - "jsesc@npm:~0.5.0": version: 0.5.0 resolution: "jsesc@npm:0.5.0" @@ -29195,6 +29882,13 @@ __metadata: languageName: node linkType: hard +"json-buffer@npm:3.0.1": + version: 3.0.1 + resolution: "json-buffer@npm:3.0.1" + checksum: 9026b03edc2847eefa2e37646c579300a1f3a4586cfb62bf857832b60c852042d0d6ae55d1afb8926163fa54c2b01d83ae24705f34990348bdac6273a29d4581 + languageName: node + linkType: hard + "json-parse-better-errors@npm:^1.0.1, json-parse-better-errors@npm:^1.0.2": version: 1.0.2 resolution: "json-parse-better-errors@npm:1.0.2" @@ -29519,6 +30213,15 @@ __metadata: languageName: node linkType: hard +"keyv@npm:^4.5.4": + version: 4.5.4 + resolution: "keyv@npm:4.5.4" + dependencies: + json-buffer: 3.0.1 + checksum: 74a24395b1c34bd44ad5cb2b49140d087553e170625240b86755a6604cd65aa16efdbdeae5cdb17ba1284a0fbb25ad06263755dbc71b8d8b06f74232ce3cdd72 + languageName: node + linkType: hard + "khroma@npm:^2.0.0": version: 2.1.0 resolution: "khroma@npm:2.1.0" @@ -32815,13 +33518,6 @@ __metadata: languageName: node linkType: hard -"node-releases@npm:^2.0.18": - version: 2.0.18 - resolution: "node-releases@npm:2.0.18" - checksum: ef55a3d853e1269a6d6279b7692cd6ff3e40bc74947945101138745bfdc9a5edabfe72cb19a31a8e45752e1910c4c65c77d931866af6357f242b172b7283f5b3 - languageName: node - linkType: hard - "node-releases@npm:^2.0.8": version: 2.0.10 resolution: "node-releases@npm:2.0.10" @@ -33462,6 +34158,18 @@ __metadata: languageName: node linkType: hard +"object.assign@npm:^4.1.5": + version: 4.1.5 + resolution: "object.assign@npm:4.1.5" + dependencies: + call-bind: ^1.0.5 + define-properties: ^1.2.1 + has-symbols: ^1.0.3 + object-keys: ^1.1.1 + checksum: f9aeac0541661370a1fc86e6a8065eb1668d3e771f7dbb33ee54578201336c057b21ee61207a186dd42db0c62201d91aac703d20d12a79fc79c353eed44d4e25 + languageName: node + linkType: hard + "object.entries@npm:^1.1.0, object.entries@npm:^1.1.2, object.entries@npm:^1.1.5": version: 1.1.5 resolution: "object.entries@npm:1.1.5" @@ -33506,6 +34214,18 @@ __metadata: languageName: node linkType: hard +"object.fromentries@npm:^2.0.8": + version: 2.0.8 + resolution: "object.fromentries@npm:2.0.8" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.2 + es-object-atoms: ^1.0.0 + checksum: 29b2207a2db2782d7ced83f93b3ff5d425f901945f3665ffda1821e30a7253cd1fd6b891a64279976098137ddfa883d748787a6fea53ecdb51f8df8b8cec0ae1 + languageName: node + linkType: hard + "object.getownpropertydescriptors@npm:^2.0.3, object.getownpropertydescriptors@npm:^2.1.0, object.getownpropertydescriptors@npm:^2.1.1, object.getownpropertydescriptors@npm:^2.1.2": version: 2.1.3 resolution: "object.getownpropertydescriptors@npm:2.1.3" @@ -33517,6 +34237,17 @@ __metadata: languageName: node linkType: hard +"object.groupby@npm:^1.0.3": + version: 1.0.3 + resolution: "object.groupby@npm:1.0.3" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.2 + checksum: 0d30693ca3ace29720bffd20b3130451dca7a56c612e1926c0a1a15e4306061d84410bdb1456be2656c5aca53c81b7a3661eceaa362db1bba6669c2c9b6d1982 + languageName: node + linkType: hard + "object.hasown@npm:^1.1.2": version: 1.1.2 resolution: "object.hasown@npm:1.1.2" @@ -33568,6 +34299,17 @@ __metadata: languageName: node linkType: hard +"object.values@npm:^1.2.0": + version: 1.2.0 + resolution: "object.values@npm:1.2.0" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-object-atoms: ^1.0.0 + checksum: 51fef456c2a544275cb1766897f34ded968b22adfc13ba13b5e4815fdaf4304a90d42a3aee114b1f1ede048a4890381d47a5594d84296f2767c6a0364b9da8fa + languageName: node + linkType: hard + "obuf@npm:^1.0.0, obuf@npm:^1.1.2": version: 1.1.2 resolution: "obuf@npm:1.1.2" @@ -34635,13 +35377,6 @@ __metadata: languageName: node linkType: hard -"picocolors@npm:^1.1.0": - version: 1.1.1 - resolution: "picocolors@npm:1.1.1" - checksum: e1cf46bf84886c79055fdfa9dcb3e4711ad259949e3565154b004b260cd356c5d54b31a1437ce9782624bf766272fe6b0154f5f0c744fb7af5d454d2b60db045 - languageName: node - linkType: hard - "picomatch@npm:^2.0.4, picomatch@npm:^2.2.1, picomatch@npm:^2.2.2, picomatch@npm:^2.2.3, picomatch@npm:^2.3.0": version: 2.3.1 resolution: "picomatch@npm:2.3.1" @@ -34929,6 +35664,13 @@ __metadata: languageName: node linkType: hard +"possible-typed-array-names@npm:^1.0.0": + version: 1.0.0 + resolution: "possible-typed-array-names@npm:1.0.0" + checksum: b32d403ece71e042385cc7856385cecf1cd8e144fa74d2f1de40d1e16035dba097bc189715925e79b67bdd1472796ff168d3a90d296356c9c94d272d5b95f3ae + languageName: node + linkType: hard + "postcss-calc@npm:^7.0.1": version: 7.0.5 resolution: "postcss-calc@npm:7.0.5" @@ -35675,6 +36417,15 @@ __metadata: languageName: node linkType: hard +"prettier@npm:^3.3.3": + version: 3.3.3 + resolution: "prettier@npm:3.3.3" + bin: + prettier: bin/prettier.cjs + checksum: bc8604354805acfdde6106852d14b045bb20827ad76a5ffc2455b71a8257f94de93f17f14e463fe844808d2ccc87248364a5691488a3304f1031326e62d9276e + languageName: node + linkType: hard + "pretty-bytes@npm:^5.6.0": version: 5.6.0 resolution: "pretty-bytes@npm:5.6.0" @@ -36996,15 +37747,6 @@ __metadata: languageName: node linkType: hard -"regexp-tree@npm:^0.1.27": - version: 0.1.27 - resolution: "regexp-tree@npm:0.1.27" - bin: - regexp-tree: bin/regexp-tree - checksum: 129aebb34dae22d6694ab2ac328be3f99105143737528ab072ef624d599afecbcfae1f5c96a166fa9e5f64fa1ecf30b411c4691e7924c3e11bbaf1712c260c54 - languageName: node - linkType: hard - "regexp.prototype.flags@npm:^1.2.0, regexp.prototype.flags@npm:^1.3.1": version: 1.4.1 resolution: "regexp.prototype.flags@npm:1.4.1" @@ -37037,6 +37779,18 @@ __metadata: languageName: node linkType: hard +"regexp.prototype.flags@npm:^1.5.2": + version: 1.5.3 + resolution: "regexp.prototype.flags@npm:1.5.3" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-errors: ^1.3.0 + set-function-name: ^2.0.2 + checksum: 83ff0705b837f7cb6d664010a11642250f36d3f642263dd0f3bdfe8f150261aa7b26b50ee97f21c1da30ef82a580bb5afedbef5f45639d69edaafbeac9bbb0ed + languageName: node + linkType: hard + "regexpp@npm:^3.0.0, regexpp@npm:^3.1.0, regexpp@npm:^3.2.0": version: 3.2.0 resolution: "regexpp@npm:3.2.0" @@ -37107,17 +37861,6 @@ __metadata: languageName: node linkType: hard -"regjsparser@npm:^0.10.0": - version: 0.10.0 - resolution: "regjsparser@npm:0.10.0" - dependencies: - jsesc: ~0.5.0 - bin: - regjsparser: bin/parser - checksum: 17550661f43ba792f8365fb95b3dbdb64e25f14e31ef7c2c11876c240a60e87b7bfc28c98589f4e76b7cf49307e45fb24d030f57d68dd0cc41c56b4d378e9254 - languageName: node - linkType: hard - "regjsparser@npm:^0.8.2": version: 0.8.4 resolution: "regjsparser@npm:0.8.4" @@ -37449,6 +38192,19 @@ resolve@1.1.7: languageName: node linkType: hard +"resolve@npm:^1.22.4": + version: 1.22.8 + resolution: "resolve@npm:1.22.8" + dependencies: + is-core-module: ^2.13.0 + path-parse: ^1.0.7 + supports-preserve-symlinks-flag: ^1.0.0 + bin: + resolve: bin/resolve + checksum: f8a26958aa572c9b064562750b52131a37c29d072478ea32e129063e2da7f83e31f7f11e7087a18225a8561cfe8d2f0df9dbea7c9d331a897571c0a2527dbb4c + languageName: node + linkType: hard + "resolve@npm:^2.0.0-next.4": version: 2.0.0-next.4 resolution: "resolve@npm:2.0.0-next.4" @@ -37508,6 +38264,19 @@ resolve@1.1.7: languageName: node linkType: hard +"resolve@patch:resolve@^1.22.4#~builtin": + version: 1.22.8 + resolution: "resolve@patch:resolve@npm%3A1.22.8#~builtin::version=1.22.8&hash=07638b" + dependencies: + is-core-module: ^2.13.0 + path-parse: ^1.0.7 + supports-preserve-symlinks-flag: ^1.0.0 + bin: + resolve: bin/resolve + checksum: 5479b7d431cacd5185f8db64bfcb7286ae5e31eb299f4c4f404ad8aa6098b77599563ac4257cb2c37a42f59dfc06a1bec2bcf283bb448f319e37f0feb9a09847 + languageName: node + linkType: hard + "resolve@patch:resolve@^2.0.0-next.4#~builtin": version: 2.0.0-next.4 resolution: "resolve@patch:resolve@npm%3A2.0.0-next.4#~builtin::version=2.0.0-next.4&hash=07638b" @@ -37876,6 +38645,18 @@ resolve@1.1.7: languageName: node linkType: hard +"safe-array-concat@npm:^1.1.2": + version: 1.1.2 + resolution: "safe-array-concat@npm:1.1.2" + dependencies: + call-bind: ^1.0.7 + get-intrinsic: ^1.2.4 + has-symbols: ^1.0.3 + isarray: ^2.0.5 + checksum: a3b259694754ddfb73ae0663829e396977b99ff21cbe8607f35a469655656da8e271753497e59da8a7575baa94d2e684bea3e10ddd74ba046c0c9b4418ffa0c4 + languageName: node + linkType: hard + "safe-buffer@npm:5.1.2, safe-buffer@npm:~5.1.0, safe-buffer@npm:~5.1.1": version: 5.1.2 resolution: "safe-buffer@npm:5.1.2" @@ -37901,6 +38682,17 @@ resolve@1.1.7: languageName: node linkType: hard +"safe-regex-test@npm:^1.0.3": + version: 1.0.3 + resolution: "safe-regex-test@npm:1.0.3" + dependencies: + call-bind: ^1.0.6 + es-errors: ^1.3.0 + is-regex: ^1.1.4 + checksum: 6c7d392ff1ae7a3ae85273450ed02d1d131f1d2c76e177d6b03eb88e6df8fa062639070e7d311802c1615f351f18dc58f9454501c58e28d5ffd9b8f502ba6489 + languageName: node + linkType: hard + "safe-regex@npm:^1.1.0": version: 1.1.0 resolution: "safe-regex@npm:1.1.0" @@ -38487,6 +39279,20 @@ resolve@1.1.7: languageName: node linkType: hard +"set-function-length@npm:^1.2.1": + version: 1.2.2 + resolution: "set-function-length@npm:1.2.2" + dependencies: + define-data-property: ^1.1.4 + es-errors: ^1.3.0 + function-bind: ^1.1.2 + get-intrinsic: ^1.2.4 + gopd: ^1.0.1 + has-property-descriptors: ^1.0.2 + checksum: a8248bdacdf84cb0fab4637774d9fb3c7a8e6089866d04c817583ff48e14149c87044ce683d7f50759a8c50fb87c7a7e173535b06169c87ef76f5fb276dfff72 + languageName: node + linkType: hard + "set-function-name@npm:^2.0.0, set-function-name@npm:^2.0.1": version: 2.0.1 resolution: "set-function-name@npm:2.0.1" @@ -38498,6 +39304,18 @@ resolve@1.1.7: languageName: node linkType: hard +"set-function-name@npm:^2.0.2": + version: 2.0.2 + resolution: "set-function-name@npm:2.0.2" + dependencies: + define-data-property: ^1.1.4 + es-errors: ^1.3.0 + functions-have-names: ^1.2.3 + has-property-descriptors: ^1.0.2 + checksum: d6229a71527fd0404399fc6227e0ff0652800362510822a291925c9d7b48a1ca1a468b11b281471c34cd5a2da0db4f5d7ff315a61d26655e77f6e971e6d0c80f + languageName: node + linkType: hard + "set-value@npm:^2.0.0, set-value@npm:^2.0.1": version: 2.0.1 resolution: "set-value@npm:2.0.1" @@ -39666,6 +40484,18 @@ resolve@1.1.7: languageName: node linkType: hard +"string.prototype.trim@npm:^1.2.9": + version: 1.2.9 + resolution: "string.prototype.trim@npm:1.2.9" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.0 + es-object-atoms: ^1.0.0 + checksum: ea2df6ec1e914c9d4e2dc856fa08228e8b1be59b59e50b17578c94a66a176888f417264bb763d4aac638ad3b3dad56e7a03d9317086a178078d131aa293ba193 + languageName: node + linkType: hard + "string.prototype.trimend@npm:^1.0.4": version: 1.0.4 resolution: "string.prototype.trimend@npm:1.0.4" @@ -39698,6 +40528,17 @@ resolve@1.1.7: languageName: node linkType: hard +"string.prototype.trimend@npm:^1.0.8": + version: 1.0.8 + resolution: "string.prototype.trimend@npm:1.0.8" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-object-atoms: ^1.0.0 + checksum: cc3bd2de08d8968a28787deba9a3cb3f17ca5f9f770c91e7e8fa3e7d47f079bad70fadce16f05dda9f261788be2c6e84a942f618c3bed31e42abc5c1084f8dfd + languageName: node + linkType: hard + "string.prototype.trimstart@npm:^1.0.4": version: 1.0.4 resolution: "string.prototype.trimstart@npm:1.0.4" @@ -39730,6 +40571,17 @@ resolve@1.1.7: languageName: node linkType: hard +"string.prototype.trimstart@npm:^1.0.8": + version: 1.0.8 + resolution: "string.prototype.trimstart@npm:1.0.8" + dependencies: + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-object-atoms: ^1.0.0 + checksum: df1007a7f580a49d692375d996521dc14fd103acda7f3034b3c558a60b82beeed3a64fa91e494e164581793a8ab0ae2f59578a49896a7af6583c1f20472bce96 + languageName: node + linkType: hard + "string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" @@ -40091,6 +40943,16 @@ resolve@1.1.7: languageName: node linkType: hard +"synckit@npm:^0.9.1": + version: 0.9.2 + resolution: "synckit@npm:0.9.2" + dependencies: + "@pkgr/core": ^0.1.0 + tslib: ^2.6.2 + checksum: 3a30e828efbdcf3b50fccab4da6e90ea7ca24d8c5c2ad3ffe98e07d7c492df121e0f75227c6e510f96f976aae76f1fa4710cb7b1d69db881caf66ef9de89360e + languageName: node + linkType: hard + "table-parser@npm:^0.1.3": version: 0.1.3 resolution: "table-parser@npm:0.1.3" @@ -41040,6 +41902,18 @@ resolve@1.1.7: languageName: node linkType: hard +"tsconfig-paths@npm:^3.15.0": + version: 3.15.0 + resolution: "tsconfig-paths@npm:3.15.0" + dependencies: + "@types/json5": ^0.0.29 + json5: ^1.0.2 + minimist: ^1.2.6 + strip-bom: ^3.0.0 + checksum: 59f35407a390d9482b320451f52a411a256a130ff0e7543d18c6f20afab29ac19fbe55c360a93d6476213cc335a4d76ce90f67df54c4e9037f7d240920832201 + languageName: node + linkType: hard + "tsconfig@npm:^7.0.0": version: 7.0.0 resolution: "tsconfig@npm:7.0.0" @@ -41073,6 +41947,13 @@ resolve@1.1.7: languageName: node linkType: hard +"tslib@npm:^2.6.2": + version: 2.8.0 + resolution: "tslib@npm:2.8.0" + checksum: de852ecd81adfdb4870927e250763345f07dc13fe7f395ce261424966bb122a0992ad844c3ec875c9e63e72afe2220a150712984e44dfd1a8a7e538a064e3d46 + languageName: node + linkType: hard + "tsup@npm:^6.5.0": version: 6.5.0 resolution: "tsup@npm:6.5.0" @@ -41259,6 +42140,17 @@ resolve@1.1.7: languageName: node linkType: hard +"typed-array-buffer@npm:^1.0.2": + version: 1.0.2 + resolution: "typed-array-buffer@npm:1.0.2" + dependencies: + call-bind: ^1.0.7 + es-errors: ^1.3.0 + is-typed-array: ^1.1.13 + checksum: 02ffc185d29c6df07968272b15d5319a1610817916ec8d4cd670ded5d1efe72901541ff2202fcc622730d8a549c76e198a2f74e312eabbfb712ed907d45cbb0b + languageName: node + linkType: hard + "typed-array-byte-length@npm:^1.0.0": version: 1.0.0 resolution: "typed-array-byte-length@npm:1.0.0" @@ -41271,6 +42163,19 @@ resolve@1.1.7: languageName: node linkType: hard +"typed-array-byte-length@npm:^1.0.1": + version: 1.0.1 + resolution: "typed-array-byte-length@npm:1.0.1" + dependencies: + call-bind: ^1.0.7 + for-each: ^0.3.3 + gopd: ^1.0.1 + has-proto: ^1.0.3 + is-typed-array: ^1.1.13 + checksum: f65e5ecd1cf76b1a2d0d6f631f3ea3cdb5e08da106c6703ffe687d583e49954d570cc80434816d3746e18be889ffe53c58bf3e538081ea4077c26a41055b216d + languageName: node + linkType: hard + "typed-array-byte-offset@npm:^1.0.0": version: 1.0.0 resolution: "typed-array-byte-offset@npm:1.0.0" @@ -41284,6 +42189,20 @@ resolve@1.1.7: languageName: node linkType: hard +"typed-array-byte-offset@npm:^1.0.2": + version: 1.0.2 + resolution: "typed-array-byte-offset@npm:1.0.2" + dependencies: + available-typed-arrays: ^1.0.7 + call-bind: ^1.0.7 + for-each: ^0.3.3 + gopd: ^1.0.1 + has-proto: ^1.0.3 + is-typed-array: ^1.1.13 + checksum: c8645c8794a621a0adcc142e0e2c57b1823bbfa4d590ad2c76b266aa3823895cf7afb9a893bf6685e18454ab1b0241e1a8d885a2d1340948efa4b56add4b5f67 + languageName: node + linkType: hard + "typed-array-length@npm:^1.0.4": version: 1.0.4 resolution: "typed-array-length@npm:1.0.4" @@ -41295,6 +42214,20 @@ resolve@1.1.7: languageName: node linkType: hard +"typed-array-length@npm:^1.0.6": + version: 1.0.6 + resolution: "typed-array-length@npm:1.0.6" + dependencies: + call-bind: ^1.0.7 + for-each: ^0.3.3 + gopd: ^1.0.1 + has-proto: ^1.0.3 + is-typed-array: ^1.1.13 + possible-typed-array-names: ^1.0.0 + checksum: f0315e5b8f0168c29d390ff410ad13e4d511c78e6006df4a104576844812ee447fcc32daab1f3a76c9ef4f64eff808e134528b5b2439de335586b392e9750e5c + languageName: node + linkType: hard + "typed-function@npm:^2.0.0": version: 2.0.0 resolution: "typed-function@npm:2.0.0" @@ -41334,6 +42267,20 @@ resolve@1.1.7: languageName: node linkType: hard +"typescript-eslint@npm:^8.11.0": + version: 8.11.0 + resolution: "typescript-eslint@npm:8.11.0" + dependencies: + "@typescript-eslint/eslint-plugin": 8.11.0 + "@typescript-eslint/parser": 8.11.0 + "@typescript-eslint/utils": 8.11.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 0dfe0445fecb86436fc285523b1d81529f31d465c3e98b48858761e4f3b1de4be106d9fb9bbf7833d63927b21cf8ddc2fc524768280145ae53b84514b05829cf + languageName: node + linkType: hard + typescript@^4.4.2: version: 4.6.3 resolution: "typescript@npm:4.6.3" @@ -41816,20 +42763,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"update-browserslist-db@npm:^1.1.0": - version: 1.1.1 - resolution: "update-browserslist-db@npm:1.1.1" - dependencies: - escalade: ^3.2.0 - picocolors: ^1.1.0 - peerDependencies: - browserslist: ">= 4.21.0" - bin: - update-browserslist-db: cli.js - checksum: 2ea11bd2562122162c3e438d83a1f9125238c0844b6d16d366e3276d0c0acac6036822dc7df65fc5a89c699cdf9f174acf439c39bedf3f9a2f3983976e4b4c3e - languageName: node - linkType: hard - "upper-case@npm:^1.1.1": version: 1.1.3 resolution: "upper-case@npm:1.1.3" @@ -43116,6 +44049,19 @@ typescript@~4.4.3: languageName: node linkType: hard +"which-typed-array@npm:^1.1.14, which-typed-array@npm:^1.1.15": + version: 1.1.15 + resolution: "which-typed-array@npm:1.1.15" + dependencies: + available-typed-arrays: ^1.0.7 + call-bind: ^1.0.7 + for-each: ^0.3.3 + gopd: ^1.0.1 + has-tostringtag: ^1.0.2 + checksum: 65227dcbfadf5677aacc43ec84356d17b5500cb8b8753059bb4397de5cd0c2de681d24e1a7bd575633f976a95f88233abfd6549c2105ef4ebd58af8aa1807c75 + languageName: node + linkType: hard + "which-typed-array@npm:^1.1.9": version: 1.1.9 resolution: "which-typed-array@npm:1.1.9" From e16a54576d5a1cb914068a6415241e9bc93a2cc8 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Sun, 27 Oct 2024 22:25:50 -0400 Subject: [PATCH 11/15] chore: Update eslint rules --- packages/cli/package.json | 2 +- packages/search/eslint.config.mjs | 72 +++---------------------------- packages/search/package.json | 3 +- yarn.lock | 20 +++------ 4 files changed, 15 insertions(+), 82 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 0e85f6074e..b8f823bdc5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -38,7 +38,7 @@ "@eslint/js": "~8.57.0", "@octokit/types": "^11.1.0", "@swc/core": "^1.3.76", - "@types/better-sqlite3": "^7.6.9", + "@types/better-sqlite3": "^7.6.11", "@types/cors": "^2.8.17", "@types/fs-extra": "^9.0.13", "@types/gitconfiglocal": "^2.0.1", diff --git a/packages/search/eslint.config.mjs b/packages/search/eslint.config.mjs index d5785f201c..4277278528 100644 --- a/packages/search/eslint.config.mjs +++ b/packages/search/eslint.config.mjs @@ -1,68 +1,10 @@ -import typescriptEslint from '@typescript-eslint/eslint-plugin'; -import eslintComments from 'eslint-plugin-eslint-comments'; -import jest from 'eslint-plugin-jest'; -import promise from 'eslint-plugin-promise'; -import globals from 'globals'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import js from '@eslint/js'; -import { FlatCompat } from '@eslint/eslintrc'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const compat = new FlatCompat({ - baseDirectory: __dirname, - recommendedConfig: js.configs.recommended, - allConfig: js.configs.all, -}); +import eslint from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import prettier from 'eslint-plugin-prettier/recommended'; export default [ - ...compat.extends( - 'plugin:@typescript-eslint/recommended', - 'plugin:@typescript-eslint/recommended-requiring-type-checking', - 'plugin:eslint-comments/recommended', - 'plugin:jest/recommended', - 'plugin:promise/recommended', - 'plugin:prettier/recommended' - ), - { - plugins: { - '@typescript-eslint': typescriptEslint, - 'eslint-comments': eslintComments, - jest, - promise, - }, - - languageOptions: { - globals: { - ...globals.node, - ...globals.browser, - ...globals.jest, - }, - - ecmaVersion: 5, - sourceType: 'commonjs', - - parserOptions: { - project: '/home/db/dev/applandinc/appmap-js/packages/search/tsconfig.json', - }, - }, - - rules: { - 'no-param-reassign': [ - 'error', - { - props: false, - }, - ], - - 'no-console': 'off', - 'no-debugger': 'off', - '@typescript-eslint/lines-between-class-members': 'off', - 'no-restricted-syntax': 'off', - 'import/no-cycle': 'off', - 'prettier/prettier': ['error'], - 'max-classes-per-file': 'off', - }, - }, + { ignores: ['built/', 'node_modules/', 'jest.config.js', 'test/fixtures'] }, + eslint.configs.recommended, + ...tseslint.configs.recommended, + prettier, ]; diff --git a/packages/search/package.json b/packages/search/package.json index e7870a287a..53ce1b3e49 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -23,7 +23,7 @@ "devDependencies": { "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.13.0", - "@types/better-sqlite3": "^7.6.9", + "@types/better-sqlite3": "^7.6.11", "@types/jest": "^29.5.4", "@types/jest-sinon": "^1.0.2", "@types/node": "^16", @@ -35,7 +35,6 @@ "eslint-plugin-jest": "^28.8.3", "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-promise": "^7.1.0", - "globals": "^15.11.0", "jest": "^29.7.0", "jest-sinon": "^1.1.0", "lint-staged": "^10.5.4", diff --git a/yarn.lock b/yarn.lock index ed0cffb34f..477bd781cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,7 +160,7 @@ __metadata: "@octokit/types": ^11.1.0 "@sidvind/better-ajv-errors": ^0.9.1 "@swc/core": ^1.3.76 - "@types/better-sqlite3": ^7.6.9 + "@types/better-sqlite3": ^7.6.11 "@types/cors": ^2.8.17 "@types/fs-extra": ^9.0.13 "@types/gitconfiglocal": ^2.0.1 @@ -592,7 +592,7 @@ __metadata: dependencies: "@eslint/eslintrc": ^3.1.0 "@eslint/js": ^9.13.0 - "@types/better-sqlite3": ^7.6.9 + "@types/better-sqlite3": ^7.6.11 "@types/jest": ^29.5.4 "@types/jest-sinon": ^1.0.2 "@types/node": ^16 @@ -605,7 +605,6 @@ __metadata: eslint-plugin-jest: ^28.8.3 eslint-plugin-prettier: ^5.2.1 eslint-plugin-promise: ^7.1.0 - globals: ^15.11.0 jest: ^29.7.0 jest-sinon: ^1.1.0 lint-staged: ^10.5.4 @@ -10879,12 +10878,12 @@ __metadata: languageName: node linkType: hard -"@types/better-sqlite3@npm:^7.6.9": - version: 7.6.9 - resolution: "@types/better-sqlite3@npm:7.6.9" +"@types/better-sqlite3@npm:^7.6.11": + version: 7.6.11 + resolution: "@types/better-sqlite3@npm:7.6.11" dependencies: "@types/node": "*" - checksum: 6572076639dde1e65ad8fe0e319e797fa40793ca805ec39aa1d072d3f145f218775eafd27d7266fb4e42a6291d8b8b836278e7880b15ef728c750dfcbba2ee52 + checksum: 981740c78f97961bf1d8e78ed8bc841792cb25695afd9d920769d5b890d40ddf3724ace8a4db1adbf634c2f05c6efe3491ce9bf66818bacd93629ea060493546 languageName: node linkType: hard @@ -24907,13 +24906,6 @@ __metadata: languageName: node linkType: hard -"globals@npm:^15.11.0": - version: 15.11.0 - resolution: "globals@npm:15.11.0" - checksum: ef32d5ef987f3d4b47fc2e389a0b235f6a46f605160c4e405722fd7b576106ca407cb867e66fd1e0fc43b631800e2e2e71847f37691026d813f96f40339da702 - languageName: node - linkType: hard - "globals@npm:^9.18.0": version: 9.18.0 resolution: "globals@npm:9.18.0" From a792c122d919ceea351812d7854b0efcbc510254 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 25 Oct 2024 11:17:12 -0400 Subject: [PATCH 12/15] chore: Upgrade dependencies Also drop unused dependencies --- packages/search/package.json | 21 +- yarn.lock | 519 +++-------------------------------- 2 files changed, 47 insertions(+), 493 deletions(-) diff --git a/packages/search/package.json b/packages/search/package.json index 53ce1b3e49..b8b5250883 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -21,13 +21,9 @@ "author": "AppLand, Inc", "license": "Commons Clause + MIT", "devDependencies": { - "@eslint/eslintrc": "^3.1.0", - "@eslint/js": "^9.13.0", "@types/better-sqlite3": "^7.6.11", "@types/jest": "^29.5.4", - "@types/jest-sinon": "^1.0.2", "@types/node": "^16", - "esbuild": "0.19.8", "eslint": "^9", "eslint-config-prettier": "^9", "eslint-plugin-eslint-comments": "^3.2.0", @@ -36,23 +32,14 @@ "eslint-plugin-prettier": "^5.2.1", "eslint-plugin-promise": "^7.1.0", "jest": "^29.7.0", - "jest-sinon": "^1.1.0", - "lint-staged": "^10.5.4", - "memfs": "^3.4.13", - "node-fetch": "2.6.7", "prettier": "^3.3.3", - "sinon": "^11.1.2", - "ts-jest": "^29.0.5", - "ts-node": "^10.9.1", - "ts-sinon": "^2.0.2", - "tsc": "^2.0.3", - "type-fest": "^3.1.0", - "typescript": "^4.9.5", + "ts-jest": "^29.2.5", + "tsc": "^2.0.4", + "typescript": "^5", "typescript-eslint": "^8.11.0" }, "dependencies": { - "better-sqlite3": "^9.5.0", - "package.json": "^2.0.1", + "better-sqlite3": "^11.5.0", "yargs": "^17.7.2" } } diff --git a/yarn.lock b/yarn.lock index 477bd781cc..acf6d6e731 100644 --- a/yarn.lock +++ b/yarn.lock @@ -590,14 +590,10 @@ __metadata: version: 0.0.0-use.local resolution: "@appland/search@workspace:packages/search" dependencies: - "@eslint/eslintrc": ^3.1.0 - "@eslint/js": ^9.13.0 "@types/better-sqlite3": ^7.6.11 "@types/jest": ^29.5.4 - "@types/jest-sinon": ^1.0.2 "@types/node": ^16 - better-sqlite3: ^9.5.0 - esbuild: 0.19.8 + better-sqlite3: ^11.5.0 eslint: ^9 eslint-config-prettier: ^9 eslint-plugin-eslint-comments: ^3.2.0 @@ -606,19 +602,10 @@ __metadata: eslint-plugin-prettier: ^5.2.1 eslint-plugin-promise: ^7.1.0 jest: ^29.7.0 - jest-sinon: ^1.1.0 - lint-staged: ^10.5.4 - memfs: ^3.4.13 - node-fetch: 2.6.7 - package.json: ^2.0.1 prettier: ^3.3.3 - sinon: ^11.1.2 - ts-jest: ^29.0.5 - ts-node: ^10.9.1 - ts-sinon: ^2.0.2 - tsc: ^2.0.3 - type-fest: ^3.1.0 - typescript: ^4.9.5 + ts-jest: ^29.2.5 + tsc: ^2.0.4 + typescript: ^5 typescript-eslint: ^8.11.0 yargs: ^17.7.2 bin: @@ -5850,7 +5837,7 @@ __metadata: languageName: node linkType: hard -"@eslint/js@npm:9.13.0, @eslint/js@npm:^9.13.0": +"@eslint/js@npm:9.13.0": version: 9.13.0 resolution: "@eslint/js@npm:9.13.0" checksum: ad5dd72aa75bd8d5bd3c1ffe68cf748aed7edef5fcf97193eb52af35dbb89a1999f526a0e2c169ef5572afbbbbb5f37d6fd0af2991d9ccdc29f753da5cc0f532 @@ -11403,15 +11390,6 @@ __metadata: languageName: node linkType: hard -"@types/keyv@npm:^3.1.1": - version: 3.1.4 - resolution: "@types/keyv@npm:3.1.4" - dependencies: - "@types/node": "*" - checksum: e009a2bfb50e90ca9b7c6e8f648f8464067271fd99116f881073fa6fa76dc8d0133181dd65e6614d5fb1220d671d67b0124aef7d97dc02d7e342ab143a47779d - languageName: node - linkType: hard - "@types/lodash@npm:^4.14.167": version: 4.14.191 resolution: "@types/lodash@npm:4.14.191" @@ -11675,15 +11653,6 @@ __metadata: languageName: node linkType: hard -"@types/responselike@npm:^1.0.0": - version: 1.0.3 - resolution: "@types/responselike@npm:1.0.3" - dependencies: - "@types/node": "*" - checksum: 6ac4b35723429b11b117e813c7acc42c3af8b5554caaf1fc750404c1ae59f9b7376bc69b9e9e194a5a97357a597c2228b7173d317320f0360d617b6425212f58 - languageName: node - linkType: hard - "@types/retry@npm:0.12.0": version: 0.12.0 resolution: "@types/retry@npm:0.12.0" @@ -14166,15 +14135,6 @@ __metadata: languageName: node linkType: hard -"abs@npm:^1.2.1": - version: 1.3.14 - resolution: "abs@npm:1.3.14" - dependencies: - ul: ^5.0.0 - checksum: af5fc49949f0694f458b7849e8ab9f10d2f6a2e1a93b3ac5cb25d62ad331d9f4d37a72fcbc1a84591fdf90fa3fffba8bf6de60dd483dd1af2200166f3436b3d3 - languageName: node - linkType: hard - "accepts@npm:~1.3.4, accepts@npm:~1.3.5, accepts@npm:~1.3.7, accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -16207,6 +16167,17 @@ __metadata: languageName: node linkType: hard +"better-sqlite3@npm:^11.5.0": + version: 11.5.0 + resolution: "better-sqlite3@npm:11.5.0" + dependencies: + bindings: ^1.5.0 + node-gyp: latest + prebuild-install: ^7.1.1 + checksum: 37acef8d4272ad57fe211aa5a2e177f95443eafb794c7db6c2d0dae0a4fe4c2ef508b8b970d82f1d9744d009677d82067279f45e27d4155a5e68a578f5c5c051 + languageName: node + linkType: hard + "better-sqlite3@npm:^9.5.0": version: 9.5.0 resolution: "better-sqlite3@npm:9.5.0" @@ -17188,13 +17159,6 @@ __metadata: languageName: node linkType: hard -"capture-stack-trace@npm:^1.0.0": - version: 1.0.2 - resolution: "capture-stack-trace@npm:1.0.2" - checksum: 13295e8176e8de74bcbe0e4fd938bed9eb4204b4cc200210ff46df91cb20b69e86f6ef42f408a59454f8b62e567ef0ee6ee5b5e7e16e686668bc77f2741542b4 - languageName: node - linkType: hard - "cardinal@npm:^2.1.1": version: 2.1.1 resolution: "cardinal@npm:2.1.1" @@ -18545,15 +18509,6 @@ __metadata: languageName: node linkType: hard -"create-error-class@npm:^3.0.1": - version: 3.0.2 - resolution: "create-error-class@npm:3.0.2" - dependencies: - capture-stack-trace: ^1.0.0 - checksum: 7254a6f96002d3226d3c1fec952473398761eb4fb12624c5dce6ed0017cdfad6de39b29aa7139680d7dcf416c25f2f308efda6eb6d9b7123f829b19ef8271511 - languageName: node - linkType: hard - "create-hash@npm:^1.1.0, create-hash@npm:^1.1.2, create-hash@npm:^1.2.0": version: 1.2.0 resolution: "create-hash@npm:1.2.0" @@ -19938,15 +19893,6 @@ __metadata: languageName: node linkType: hard -"deffy@npm:^2.2.1, deffy@npm:^2.2.2": - version: 2.2.4 - resolution: "deffy@npm:2.2.4" - dependencies: - typpy: ^2.0.0 - checksum: a06f44306c676d5fb663120610060a3abc48c745200f306a11acc2936095f901b170018fa5ffedbc1ebddf43b83467dc11b90b9ec5fdd232f970d85d66515543 - languageName: node - linkType: hard - "define-data-property@npm:^1.0.1, define-data-property@npm:^1.1.1": version: 1.1.1 resolution: "define-data-property@npm:1.1.1" @@ -20646,7 +20592,7 @@ __metadata: languageName: node linkType: hard -"duplexer2@npm:^0.1.4, duplexer2@npm:~0.1.0": +"duplexer2@npm:~0.1.0": version: 0.1.4 resolution: "duplexer2@npm:0.1.4" dependencies: @@ -21019,15 +20965,6 @@ __metadata: languageName: node linkType: hard -"err@npm:^1.1.1": - version: 1.1.1 - resolution: "err@npm:1.1.1" - dependencies: - typpy: ^2.2.0 - checksum: e9b8b5226724e32c7fdf2a14c1380093e130e1bfa4e15f6886ffd7239f0a51b8733da7b8790f01a6681f641a97838d0537e18d553528ab52245a05e6dcf75fe7 - languageName: node - linkType: hard - "errno@npm:^0.1.1, errno@npm:^0.1.3, errno@npm:~0.1.7": version: 0.1.8 resolution: "errno@npm:0.1.8" @@ -21039,7 +20976,7 @@ __metadata: languageName: node linkType: hard -"error-ex@npm:^1.2.0, error-ex@npm:^1.3.1": +"error-ex@npm:^1.3.1": version: 1.3.2 resolution: "error-ex@npm:1.3.2" dependencies: @@ -22926,16 +22863,6 @@ __metadata: languageName: node linkType: hard -"exec-limiter@npm:^3.0.0": - version: 3.2.13 - resolution: "exec-limiter@npm:3.2.13" - dependencies: - limit-it: ^3.0.0 - typpy: ^2.1.0 - checksum: 61c4d7e222bf5d062e9bf9a067cc9346c6d16cce935e5e4ddaa0c2fbcbda9b214545f04aeabc3bb902a17b0b340b6710f4e7ead4999ed255764c028074a1b3c7 - languageName: node - linkType: hard - "exec-sh@npm:^0.3.2": version: 0.3.6 resolution: "exec-sh@npm:0.3.6" @@ -24263,15 +24190,6 @@ __metadata: languageName: node linkType: hard -"function.name@npm:^1.0.3": - version: 1.0.13 - resolution: "function.name@npm:1.0.13" - dependencies: - noop6: ^1.0.1 - checksum: 376bd4247cacffb50ae425c64e99c42fd10a875197e09db7e65002a5bc0f539608b03f1ff7e27234516cd5b3bbfd9d9d8e912fa9aaeec52d3b422d501f290fff - languageName: node - linkType: hard - "function.prototype.name@npm:^1.1.0, function.prototype.name@npm:^1.1.5": version: 1.1.5 resolution: "function.prototype.name@npm:1.1.5" @@ -24604,51 +24522,6 @@ __metadata: languageName: node linkType: hard -"git-package-json@npm:^1.4.0": - version: 1.4.10 - resolution: "git-package-json@npm:1.4.10" - dependencies: - deffy: ^2.2.1 - err: ^1.1.1 - gry: ^5.0.0 - normalize-package-data: ^2.3.5 - oargv: ^3.4.1 - one-by-one: ^3.1.0 - r-json: ^1.2.1 - r-package-json: ^1.0.0 - tmp: 0.0.28 - checksum: 69c16f42e9dfbfd60daf5f7cf33642b2cf4ebf90ffac452b4eacf12c3d9f1728da1c0d771348f6adccd7f4c4a3a7f271e19fbfbb81e16c4096143dfe7ad7df4a - languageName: node - linkType: hard - -"git-source@npm:^1.1.0": - version: 1.1.10 - resolution: "git-source@npm:1.1.10" - dependencies: - git-url-parse: ^5.0.1 - checksum: d3efbe1b32994c82827b43fa5e5de2503bf5e7f44c048b27514d6617f6e0119f4f609c899ad261c63661749867938767bff0a5180c2a3e6193ba386e497e17be - languageName: node - linkType: hard - -"git-up@npm:^1.0.0": - version: 1.2.1 - resolution: "git-up@npm:1.2.1" - dependencies: - is-ssh: ^1.0.0 - parse-url: ^1.0.0 - checksum: 45a939df02e949a1608858b472b50d7165a17521ded288365d0c5b0eb8fa157ba14a6ceaeda316ebdcf4c1c455bbd037a86cc48432723d8c094404d881357b0d - languageName: node - linkType: hard - -"git-url-parse@npm:^5.0.1": - version: 5.0.1 - resolution: "git-url-parse@npm:5.0.1" - dependencies: - git-up: ^1.0.0 - checksum: 081c37feac708e93601ab8b0c6c5b465984dd62334e0f5121ca95f4d4d22fa668f97ade78ed5fb42f3b637ab3c60785eae2e60402e5c356fae97292791d425b1 - languageName: node - linkType: hard - "gitconfiglocal@npm:^2.1.0": version: 2.1.0 resolution: "gitconfiglocal@npm:2.1.0" @@ -24997,29 +24870,6 @@ __metadata: languageName: node linkType: hard -"got@npm:^5.0.0": - version: 5.7.1 - resolution: "got@npm:5.7.1" - dependencies: - create-error-class: ^3.0.1 - duplexer2: ^0.1.4 - is-redirect: ^1.0.0 - is-retry-allowed: ^1.0.0 - is-stream: ^1.0.0 - lowercase-keys: ^1.0.0 - node-status-codes: ^1.0.0 - object-assign: ^4.0.1 - parse-json: ^2.1.0 - pinkie-promise: ^2.0.0 - read-all-stream: ^3.0.0 - readable-stream: ^2.0.5 - timed-out: ^3.0.0 - unzip-response: ^1.0.2 - url-parse-lax: ^1.0.0 - checksum: bd8e0d0b28e27bc34b81ce98a7ff116b2dec77ac70dda9cf624000eca303ebff51a7c00c1113bede3038afa2e865a61e6a5a6d2c645c5adf68ac8ed5e5b8d064 - languageName: node - linkType: hard - "graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.15, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.4, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.9 resolution: "graceful-fs@npm:4.2.9" @@ -25071,18 +24921,6 @@ __metadata: languageName: node linkType: hard -"gry@npm:^5.0.0": - version: 5.0.8 - resolution: "gry@npm:5.0.8" - dependencies: - abs: ^1.2.1 - exec-limiter: ^3.0.0 - one-by-one: ^3.0.0 - ul: ^5.0.0 - checksum: 969eb7a98f8e5f290e2fab9e689a2cef02099ef92abceafdc9d7074356e34e2ab275a56f148867ead9d7db79b6dc7458a8208b805c4a9dbd43b5a406d5402d7c - languageName: node - linkType: hard - "gunzip-maybe@npm:^1.4.2": version: 1.4.2 resolution: "gunzip-maybe@npm:1.4.2" @@ -27078,13 +26916,6 @@ __metadata: languageName: node linkType: hard -"is-redirect@npm:^1.0.0": - version: 1.0.0 - resolution: "is-redirect@npm:1.0.0" - checksum: 25dd3d9943f57ef0f29d28e2d9deda8288e0c7098ddc65abec3364ced9a6491ea06cfaf5110c61fc40ec1fde706b73cee5d171f85278edbf4e409b85725bfea7 - languageName: node - linkType: hard - "is-reference@npm:^1.1.2, is-reference@npm:^1.2.1": version: 1.2.1 resolution: "is-reference@npm:1.2.1" @@ -27118,13 +26949,6 @@ __metadata: languageName: node linkType: hard -"is-retry-allowed@npm:^1.0.0": - version: 1.2.0 - resolution: "is-retry-allowed@npm:1.2.0" - checksum: 50d700a89ae31926b1c91b3eb0104dbceeac8790d8b80d02f5c76d9a75c2056f1bb24b5268a8a018dead606bddf116b2262e5ac07401eb8b8783b266ed22558d - languageName: node - linkType: hard - "is-set@npm:^2.0.1, is-set@npm:^2.0.2": version: 2.0.2 resolution: "is-set@npm:2.0.2" @@ -27157,16 +26981,7 @@ __metadata: languageName: node linkType: hard -"is-ssh@npm:^1.0.0, is-ssh@npm:^1.3.0": - version: 1.4.0 - resolution: "is-ssh@npm:1.4.0" - dependencies: - protocols: ^2.0.1 - checksum: 75eaa17b538bee24b661fbeb0f140226ac77e904a6039f787bea418431e2162f1f9c4c4ccad3bd169e036cd701cc631406e8c505d9fa7e20164e74b47f86f40f - languageName: node - linkType: hard - -"is-stream@npm:^1.0.0, is-stream@npm:^1.1.0": +"is-stream@npm:^1.1.0": version: 1.1.0 resolution: "is-stream@npm:1.1.0" checksum: 063c6bec9d5647aa6d42108d4c59723d2bd4ae42135a2d4db6eadbd49b7ea05b750fd69d279e5c7c45cf9da753ad2c00d8978be354d65aa9f6bb434969c6a2ae @@ -27529,13 +27344,6 @@ __metadata: languageName: node linkType: hard -"iterate-object@npm:^1.1.0": - version: 1.3.4 - resolution: "iterate-object@npm:1.3.4" - checksum: b63496c489177babccb4b487322279ea4377e08d02b93902c3ffba3032a788f014a74e03c615da2a24807fa3fca872f69c9570f7801c8e88181df7a49298904b - languageName: node - linkType: hard - "iterate-value@npm:^1.0.2": version: 1.0.2 resolution: "iterate-value@npm:1.0.2" @@ -31044,15 +30852,6 @@ __metadata: languageName: node linkType: hard -"limit-it@npm:^3.0.0": - version: 3.2.10 - resolution: "limit-it@npm:3.2.10" - dependencies: - typpy: ^2.0.0 - checksum: 3a809aad23f191d2abd469d835829391c8e85074de109a25af44c2d939c41fc92643ab8190f362191bd9157119f32574cebbc7eef6e99695f354a01f7b2a7b76 - languageName: node - linkType: hard - "lines-and-columns@npm:^1.1.6": version: 1.2.4 resolution: "lines-and-columns@npm:1.2.4" @@ -31465,13 +31264,6 @@ __metadata: languageName: node linkType: hard -"lowercase-keys@npm:^1.0.0": - version: 1.0.1 - resolution: "lowercase-keys@npm:1.0.1" - checksum: 4d045026595936e09953e3867722e309415ff2c80d7701d067546d75ef698dac218a4f53c6d1d0e7368b47e45fd7529df47e6cb56fbb90523ba599f898b3d147 - languageName: node - linkType: hard - "lru-cache@npm:10.2.2, lru-cache@npm:^10.2.2": version: 10.2.2 resolution: "lru-cache@npm:10.2.2" @@ -33517,13 +33309,6 @@ __metadata: languageName: node linkType: hard -"node-status-codes@npm:^1.0.0": - version: 1.0.0 - resolution: "node-status-codes@npm:1.0.0" - checksum: 10fe52de31cc94536aa49a2a8a28e39a880d02832ac268e7edd2b082292232abcaa8e44fe4a318d072a08ce114851fab269ab8d7f9527bd5609aebaf2bb6df17 - languageName: node - linkType: hard - "non-layered-tidy-tree-layout@npm:^2.0.2": version: 2.0.2 resolution: "non-layered-tidy-tree-layout@npm:2.0.2" @@ -33531,13 +33316,6 @@ __metadata: languageName: node linkType: hard -"noop6@npm:^1.0.1": - version: 1.0.9 - resolution: "noop6@npm:1.0.9" - checksum: cc46d03eb22c5a990b3ce5e3ff71a82628efadcdeefdae566639657d2a947033ea919d1f4bfc7bfbcf674d43c21b3e580c2f07c8043e7ef487ac527237e2532f - languageName: node - linkType: hard - "nopt@npm:^5.0.0": version: 5.0.0 resolution: "nopt@npm:5.0.0" @@ -33571,7 +33349,7 @@ __metadata: languageName: node linkType: hard -"normalize-package-data@npm:^2.3.2, normalize-package-data@npm:^2.3.5, normalize-package-data@npm:^2.5.0": +"normalize-package-data@npm:^2.3.2, normalize-package-data@npm:^2.5.0": version: 2.5.0 resolution: "normalize-package-data@npm:2.5.0" dependencies: @@ -34035,16 +33813,6 @@ __metadata: languageName: node linkType: hard -"oargv@npm:^3.4.1": - version: 3.4.10 - resolution: "oargv@npm:3.4.10" - dependencies: - iterate-object: ^1.1.0 - ul: ^5.0.0 - checksum: f713a1995da354236ec87e8f4dcc21326fe3289129a2c3731b868432b845043ffc4a85ce390c734e25c99e2a8650b745b25282c5b4dffc61496d7f7432dd924d - languageName: node - linkType: hard - "oauth-sign@npm:~0.9.0": version: 0.9.0 resolution: "oauth-sign@npm:0.9.0" @@ -34052,15 +33820,6 @@ __metadata: languageName: node linkType: hard -"obj-def@npm:^1.0.0": - version: 1.0.9 - resolution: "obj-def@npm:1.0.9" - dependencies: - deffy: ^2.2.2 - checksum: 93e28098d186a5609968bf0aed4d0816f0e5baf0354093a6f79687d9e419bd26c38ff6a5674b009ed22987634cb915936cf077f5180a5dbd9e824c80a74c0064 - languageName: node - linkType: hard - "object-assign@npm:^4, object-assign@npm:^4.0.1, object-assign@npm:^4.1.0, object-assign@npm:^4.1.1": version: 4.1.1 resolution: "object-assign@npm:4.1.1" @@ -34359,16 +34118,6 @@ __metadata: languageName: node linkType: hard -"one-by-one@npm:^3.0.0, one-by-one@npm:^3.1.0": - version: 3.2.8 - resolution: "one-by-one@npm:3.2.8" - dependencies: - obj-def: ^1.0.0 - sliced: ^1.0.1 - checksum: 0af8cef27306172a67423001a1b8b242115ab733d18890275c9cd4adc8ec3414f0b453395915ca8eaddffbbbf71a68e74d16eded362dbd8fac22365af81277e1 - languageName: node - linkType: hard - "onetime@npm:^2.0.0": version: 2.0.1 resolution: "onetime@npm:2.0.1" @@ -34594,7 +34343,7 @@ __metadata: languageName: node linkType: hard -"os-tmpdir@npm:^1.0.1, os-tmpdir@npm:~1.0.1, os-tmpdir@npm:~1.0.2": +"os-tmpdir@npm:^1.0.1, os-tmpdir@npm:~1.0.2": version: 1.0.2 resolution: "os-tmpdir@npm:1.0.2" checksum: 5666560f7b9f10182548bf7013883265be33620b1c1b4a4d405c25be2636f970c5488ff3e6c48de75b55d02bde037249fe5dbfbb4c0fb7714953d56aed062e6d @@ -34845,38 +34594,6 @@ __metadata: languageName: node linkType: hard -"package-json-path@npm:^1.0.0": - version: 1.0.9 - resolution: "package-json-path@npm:1.0.9" - dependencies: - abs: ^1.2.1 - checksum: 4528ba905217628e5abf8bf843e7a5c6a9bb368068105ecca8c9d7d925922b5b9f982295710b0df63e1a51705b8077df449252a9fc17765e3b7e0358ae476860 - languageName: node - linkType: hard - -"package-json@npm:^2.3.1": - version: 2.4.0 - resolution: "package-json@npm:2.4.0" - dependencies: - got: ^5.0.0 - registry-auth-token: ^3.0.1 - registry-url: ^3.0.3 - semver: ^5.1.0 - checksum: 0120e4e8222e796f0b7c3e23c25d02a6cbb4818f4488f97e9dcd619383d56c8a7e7d9a0e46dee6a28b56ecabdbfa07ffca0049bff268ebf0ded0e7f1d906e1a0 - languageName: node - linkType: hard - -"package.json@npm:^2.0.1": - version: 2.0.1 - resolution: "package.json@npm:2.0.1" - dependencies: - git-package-json: ^1.4.0 - git-source: ^1.1.0 - package-json: ^2.3.1 - checksum: 579c4c3d6bdeb67e6e501bbaca82ee0a8c9590bf23c6da9eec8f7c2a4dcb3dc0b5dce9ce90ccfcde4c235c8269e1cebd84693d5123d16e5c1a0a22152579f6b6 - languageName: node - linkType: hard - "pacote@npm:^13.0.3": version: 13.1.1 resolution: "pacote@npm:13.1.1" @@ -35053,15 +34770,6 @@ __metadata: languageName: node linkType: hard -"parse-json@npm:^2.1.0": - version: 2.2.0 - resolution: "parse-json@npm:2.2.0" - dependencies: - error-ex: ^1.2.0 - checksum: dda78a63e57a47b713a038630868538f718a7ca0cd172a36887b0392ccf544ed0374902eb28f8bf3409e8b71d62b79d17062f8543afccf2745f9b0b2d2bb80ca - languageName: node - linkType: hard - "parse-json@npm:^4.0.0": version: 4.0.0 resolution: "parse-json@npm:4.0.0" @@ -35091,16 +34799,6 @@ __metadata: languageName: node linkType: hard -"parse-url@npm:^1.0.0": - version: 1.3.11 - resolution: "parse-url@npm:1.3.11" - dependencies: - is-ssh: ^1.3.0 - protocols: ^1.4.0 - checksum: 33e36566ed248cc8289a35c6c094f98cf68ee42735bd971f9696db4e75d4d85889e4cefecb4fbfab1da4990854e0a5b548434726aede7b9c72714d9dc45736c7 - languageName: node - linkType: hard - "parse5-htmlparser2-tree-adapter@npm:^6.0.0": version: 6.0.1 resolution: "parse5-htmlparser2-tree-adapter@npm:6.0.1" @@ -36359,7 +36057,7 @@ __metadata: languageName: node linkType: hard -"prepend-http@npm:^1.0.0, prepend-http@npm:^1.0.1": +"prepend-http@npm:^1.0.0": version: 1.0.4 resolution: "prepend-http@npm:1.0.4" checksum: 01e7baf4ad38af02257b99098543469332fc42ae50df33d97a124bf8172295907352fa6138c9b1610c10c6dd0847ca736e53fda736387cc5cf8fcffe96b47f29 @@ -36678,20 +36376,6 @@ __metadata: languageName: node linkType: hard -"protocols@npm:^1.4.0": - version: 1.4.8 - resolution: "protocols@npm:1.4.8" - checksum: 2d555c013df0b05402970f67f7207c9955a92b1d13ffa503c814b5fe2f6dde7ac6a03320e0975c1f5832b0113327865e0b3b28bfcad023c25ddb54b53fab8684 - languageName: node - linkType: hard - -"protocols@npm:^2.0.1": - version: 2.0.1 - resolution: "protocols@npm:2.0.1" - checksum: 4a9bef6aa0449a0245ded319ac3cbfd032c3e76ebb562777037a3a832c99253d0e8bc2847f7be350236df620a11f7d4fe683ea7f59a2cc14c69f746b6259eda4 - languageName: node - linkType: hard - "proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" @@ -37149,25 +36833,6 @@ __metadata: languageName: node linkType: hard -"r-json@npm:^1.2.1": - version: 1.3.0 - resolution: "r-json@npm:1.3.0" - dependencies: - w-json: 1.3.10 - checksum: 9a2aa9b92a2f4b7932c7eb45175d9c7ff078e322eecaf1ca2c9cdda346ea68e73062004c1b3631a9127e84eedf982fc816110f0c7a1d07c6b2b3344f6d621791 - languageName: node - linkType: hard - -"r-package-json@npm:^1.0.0": - version: 1.0.9 - resolution: "r-package-json@npm:1.0.9" - dependencies: - package-json-path: ^1.0.0 - r-json: ^1.2.1 - checksum: e94b2b02e75dee37d42226656a66cacf69332324acf67743ddf4aa8300acc06bc35861c15379a4061232ca40d40d76f5a1beb82fe18b9e5d6225e2b722a3a138 - languageName: node - linkType: hard - "ramda@npm:0.29.0": version: 0.29.0 resolution: "ramda@npm:0.29.0" @@ -37262,7 +36927,7 @@ __metadata: languageName: node linkType: hard -"rc@npm:^1.0.1, rc@npm:^1.1.6, rc@npm:^1.2.7, rc@npm:^1.2.8": +"rc@npm:^1.2.7, rc@npm:^1.2.8": version: 1.2.8 resolution: "rc@npm:1.2.8" dependencies: @@ -37380,16 +37045,6 @@ __metadata: languageName: node linkType: hard -"read-all-stream@npm:^3.0.0": - version: 3.1.0 - resolution: "read-all-stream@npm:3.1.0" - dependencies: - pinkie-promise: ^2.0.0 - readable-stream: ^2.0.0 - checksum: ff7bf7c5484dcb6e857d9eccc388d2d32674e538c88cad61703e3d8ae9386a7fa74e8b30b6eb82a3979ea60413e933b90238e99a59f782d7fabebb7833a5b9c4 - languageName: node - linkType: hard - "read-cmd-shim@npm:^3.0.0": version: 3.0.1 resolution: "read-cmd-shim@npm:3.0.1" @@ -37510,21 +37165,6 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.5": - version: 2.3.8 - resolution: "readable-stream@npm:2.3.8" - dependencies: - core-util-is: ~1.0.0 - inherits: ~2.0.3 - isarray: ~1.0.0 - process-nextick-args: ~2.0.0 - safe-buffer: ~5.1.1 - string_decoder: ~1.1.1 - util-deprecate: ~1.0.1 - checksum: 65645467038704f0c8aaf026a72fbb588a9e2ef7a75cd57a01702ee9db1c4a1e4b03aaad36861a6a0926546a74d174149c8c207527963e0c2d3eee2f37678a42 - languageName: node - linkType: hard - "readdir-scoped-modules@npm:^1.1.0": version: 1.1.0 resolution: "readdir-scoped-modules@npm:1.1.0" @@ -37818,16 +37458,6 @@ __metadata: languageName: node linkType: hard -"registry-auth-token@npm:^3.0.1": - version: 3.4.0 - resolution: "registry-auth-token@npm:3.4.0" - dependencies: - rc: ^1.1.6 - safe-buffer: ^5.0.1 - checksum: a15780726bae327a8fff4048cb6a5de03d58bc19ea9e2411322e32e4ebb59962efb669d270bdde384ed68ed7b948f5feb11469e3d0c7e50a33cc8866710f0bc2 - languageName: node - linkType: hard - "registry-auth-token@npm:^4.0.0": version: 4.2.1 resolution: "registry-auth-token@npm:4.2.1" @@ -37837,15 +37467,6 @@ __metadata: languageName: node linkType: hard -"registry-url@npm:^3.0.3": - version: 3.1.0 - resolution: "registry-url@npm:3.1.0" - dependencies: - rc: ^1.0.1 - checksum: 6d223da41b04e1824f5faa63905c6f2e43b216589d72794111573f017352b790aef42cd1f826463062f89d804abb2027e3d9665d2a9a0426a11eedd04d470af3 - languageName: node - linkType: hard - "regjsgen@npm:^0.6.0": version: 0.6.0 resolution: "regjsgen@npm:0.6.0" @@ -39053,15 +38674,6 @@ resolve@1.1.7: languageName: node linkType: hard -"semver@npm:^5.1.0": - version: 5.7.2 - resolution: "semver@npm:5.7.2" - bin: - semver: bin/semver - checksum: fb4ab5e0dd1c22ce0c937ea390b4a822147a9c53dbd2a9a0132f12fe382902beef4fbf12cf51bb955248d8d15874ce8cd89532569756384f994309825f10b686 - languageName: node - linkType: hard - "semver@npm:^6.0.0, semver@npm:^6.1.0, semver@npm:^6.1.1, semver@npm:^6.1.2, semver@npm:^6.2.0, semver@npm:^6.3.0": version: 6.3.0 resolution: "semver@npm:6.3.0" @@ -39616,13 +39228,6 @@ resolve@1.1.7: languageName: node linkType: hard -"sliced@npm:^1.0.1": - version: 1.0.1 - resolution: "sliced@npm:1.0.1" - checksum: 84528d23279985ead75809eeec5d601b0fb6bc28348c6627f4feb40747533a1e36a75e8bc60f9079528079b21c434890b397e8fc5c24a649165cc0bbe90b4d70 - languageName: node - linkType: hard - "smart-buffer@npm:^4.2.0": version: 4.2.0 resolution: "smart-buffer@npm:4.2.0" @@ -41310,13 +40915,6 @@ resolve@1.1.7: languageName: node linkType: hard -"timed-out@npm:^3.0.0": - version: 3.1.3 - resolution: "timed-out@npm:3.1.3" - checksum: 7952bcc926fd43f3206f7cb9f27dc54912a48c08058da0208d33dabef35b6759d9f1d2d14976c529d2209bd92a109bb057d377211fd2d2d962f4ca0a8d3f15f0 - languageName: node - linkType: hard - "timers-browserify@npm:^2.0.4": version: 2.0.12 resolution: "timers-browserify@npm:2.0.12" @@ -41363,15 +40961,6 @@ resolve@1.1.7: languageName: node linkType: hard -"tmp@npm:0.0.28": - version: 0.0.28 - resolution: "tmp@npm:0.0.28" - dependencies: - os-tmpdir: ~1.0.1 - checksum: 8167d2471b650f88fde1515bbf6c62d2adef07972d06531569f789863a2436c32027b6d5fbe3102ed2c430b86043dffd337db12494f1f982534862d9487e4eff - languageName: node - linkType: hard - "tmp@npm:^0.0.33": version: 0.0.33 resolution: "tmp@npm:0.0.33" @@ -42303,6 +41892,16 @@ typescript@^4.5.4: languageName: node linkType: hard +"typescript@npm:^5": + version: 5.6.3 + resolution: "typescript@npm:5.6.3" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: ba302f8822777ebefb28b554105f3e074466b671e7444ec6b75dadc008a62f46f373d9e57ceced1c433756d06c8b7dc569a7eefdf3a9573122a49205ff99021a + languageName: node + linkType: hard + "typescript@npm:^5.3.2": version: 5.3.3 resolution: "typescript@npm:5.3.3" @@ -42343,6 +41942,16 @@ typescript@^4.5.4: languageName: node linkType: hard +"typescript@patch:typescript@^5#~builtin": + version: 5.6.3 + resolution: "typescript@patch:typescript@npm%3A5.6.3#~builtin::version=5.6.3&hash=7ad353" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: ade87bce2363ee963eed0e4ca8a312ea02c81873ebd53609bc3f6dc0a57f6e61ad7e3fb8cbb7f7ab8b5081cbee801b023f7c4823ee70b1c447eae050e6c7622b + languageName: node + linkType: hard + "typescript@patch:typescript@^5.3.2#~builtin": version: 5.3.3 resolution: "typescript@patch:typescript@npm%3A5.3.3#~builtin::version=5.3.3&hash=7ad353" @@ -42373,15 +41982,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"typpy@npm:^2.0.0, typpy@npm:^2.1.0, typpy@npm:^2.2.0, typpy@npm:^2.3.4": - version: 2.3.13 - resolution: "typpy@npm:2.3.13" - dependencies: - function.name: ^1.0.3 - checksum: e415782245876bd2bd069e227feb93fb0e4d93ccc4c1f10c0bd2a2ba223a92dfca18a4ed95b8c87eb226337909f4fa1509c6e962d7917f6d648e7d24b497a13e - languageName: node - linkType: hard - "uglify-js@npm:3.4.x": version: 3.4.10 resolution: "uglify-js@npm:3.4.10" @@ -42403,16 +42003,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"ul@npm:^5.0.0": - version: 5.2.15 - resolution: "ul@npm:5.2.15" - dependencies: - deffy: ^2.2.2 - typpy: ^2.3.4 - checksum: a47735d307da2e9b25568977d8b13d5ab2845b2f7a14ad75b22022945f3879e0ecfef8b6134cb661fc2e46a2e86e3f69c44659816c422d0be50181aeaf44413d - languageName: node - linkType: hard - "unbox-primitive@npm:^1.0.1": version: 1.0.1 resolution: "unbox-primitive@npm:1.0.1" @@ -42699,13 +42289,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"unzip-response@npm:^1.0.2": - version: 1.0.2 - resolution: "unzip-response@npm:1.0.2" - checksum: 09efe5d1d23a40534f5f67f268c6f4d2533cba4f2d40ae3233e50f9d7d3ee1db5503c52014f3265174517fff9fdebedd9ac6ec0c57b23a4933791d754b722187 - languageName: node - linkType: hard - "upath@npm:^1.1.1": version: 1.2.0 resolution: "upath@npm:1.2.0" @@ -42802,15 +42385,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"url-parse-lax@npm:^1.0.0": - version: 1.0.0 - resolution: "url-parse-lax@npm:1.0.0" - dependencies: - prepend-http: ^1.0.1 - checksum: 03316acff753845329652258c16d1688765ee34f7d242a94dadf9ff6e43ea567ec062cec7aa27c37f76f2c57f95e0660695afff32fb97b527591c7340a3090fa - languageName: node - linkType: hard - "url-parse@npm:^1.5.3": version: 1.5.10 resolution: "url-parse@npm:1.5.10" @@ -43371,13 +42945,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"w-json@npm:1.3.10": - version: 1.3.10 - resolution: "w-json@npm:1.3.10" - checksum: 8535a207e579e616797efc4d5140acc7c0aefd11f0c9f846e6739816a2db8637d235492d86fc5c47bb2dba5821413d72b2d62df9184ee9d6e22e67b3f90d205b - languageName: node - linkType: hard - "w3c-hr-time@npm:^1.0.1, w3c-hr-time@npm:^1.0.2": version: 1.0.2 resolution: "w3c-hr-time@npm:1.0.2" From 3604ea8a81dac666ec589f7d582091387ff76671 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 25 Oct 2024 11:17:42 -0400 Subject: [PATCH 13/15] refactor: Fix up eslint errors --- packages/search/src/build-file-index.ts | 8 ++++++-- packages/search/src/cli.ts | 12 +++++++++--- packages/search/src/file-type.ts | 1 + packages/search/src/git.ts | 2 +- packages/search/src/index.ts | 2 +- packages/search/src/ioutil.ts | 1 + packages/search/src/snippet-index.ts | 13 +++++++++++-- packages/search/src/split-camelized.ts | 6 +++--- packages/search/src/splitter.ts | 14 ++------------ 9 files changed, 35 insertions(+), 24 deletions(-) diff --git a/packages/search/src/build-file-index.ts b/packages/search/src/build-file-index.ts index 99e512ae4b..b17f993160 100644 --- a/packages/search/src/build-file-index.ts +++ b/packages/search/src/build-file-index.ts @@ -2,7 +2,8 @@ import makeDebug from 'debug'; import { join } from 'path'; import FileIndex from './file-index'; -import { ContentReader, readFileSafe } from './ioutil'; +import { ContentReader } from './ioutil'; +import { warn } from 'console'; export type ListFn = (path: string) => Promise; @@ -44,7 +45,10 @@ async function indexDirectory(context: Context, directory: string) { debug('Indexing: %s', filePath); if (await context.fileFilter(filePath)) { - indexFile(context, filePath); + indexFile(context, filePath).catch((e) => { + warn(`Error indexing file: ${filePath}`); + warn(e); + }); } } } diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts index dbe0baa54b..515ba712ad 100644 --- a/packages/search/src/cli.ts +++ b/packages/search/src/cli.ts @@ -15,8 +15,7 @@ import { langchainSplitter } from './splitter'; import assert from 'assert'; const debug = makeDebug('appmap:search:cli'); - -yargs(hideBin(process.argv)) +const cli = yargs(hideBin(process.argv)) .command( '* ', 'Index directories and perform a search', @@ -101,7 +100,7 @@ yargs(hideBin(process.argv)) console.log('Snippet search results'); console.log('----------------------'); - const isNullOrUndefined = (value: any) => value === null || value === undefined; + const isNullOrUndefined = (value: unknown) => value === null || value === undefined; const snippetSearchResults = snippetIndex.searchSnippets(query as string); for (const result of snippetSearchResults) { @@ -124,3 +123,10 @@ yargs(hideBin(process.argv)) } ) .help().argv; + +if (cli instanceof Promise) { + cli.catch((e) => { + console.error(e); + process.exit(1); + }); +} diff --git a/packages/search/src/file-type.ts b/packages/search/src/file-type.ts index a4e044efb7..2f2887eb71 100644 --- a/packages/search/src/file-type.ts +++ b/packages/search/src/file-type.ts @@ -91,6 +91,7 @@ const statFileSafe = async (filePath: string): Promise => { return stats.size; } catch (error) { debug(`Error reading file: %s`, filePath); + debug(error); return undefined; } }; diff --git a/packages/search/src/git.ts b/packages/search/src/git.ts index 0334c80ca6..79c4cfbe53 100644 --- a/packages/search/src/git.ts +++ b/packages/search/src/git.ts @@ -160,7 +160,7 @@ export const Git = new Proxy(GitProperties, { if (gitCache.has(cacheKey)) { return gitCache.get(cacheKey); } - /* eslint-disable-next-line @typescript-eslint/ban-types */ + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type const result: unknown = Reflect.apply(target as Function, thisArg, argArray); gitCache.set(cacheKey, result); return result; diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts index ffe8ebb5f6..05fc5fe24b 100644 --- a/packages/search/src/index.ts +++ b/packages/search/src/index.ts @@ -1,5 +1,5 @@ export { ContentReader, readFileSafe } from './ioutil'; -export { Splitter, langchainSplitter, wohleDocumentSplitter } from './splitter'; +export { Splitter, langchainSplitter } from './splitter'; export { ListFn, FilterFn, Tokenizer, default as buildFileIndex } from './build-file-index'; export { File, default as buildSnippetIndex } from './build-snippet-index'; export { default as SnippetIndex, SnippetSearchResult } from './snippet-index'; diff --git a/packages/search/src/ioutil.ts b/packages/search/src/ioutil.ts index 0176171447..e20d37e935 100644 --- a/packages/search/src/ioutil.ts +++ b/packages/search/src/ioutil.ts @@ -10,6 +10,7 @@ export function readFileSafe(filePath: string): PromiseLike return Promise.resolve(readFileSync(filePath, 'utf8')); } catch (error) { debug(`Error reading file: %s`, filePath); + debug(error); return Promise.resolve(undefined); } } diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts index 81986af2ab..d30672cce1 100644 --- a/packages/search/src/snippet-index.ts +++ b/packages/search/src/snippet-index.ts @@ -1,5 +1,4 @@ import sqlite3 from 'better-sqlite3'; -import { warn } from 'console'; const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5( snippet_id UNINDEXED, @@ -58,6 +57,16 @@ export type SnippetSearchResult = { content: string; }; +type SnippetSearchRow = { + snippet_id: string; + directory: string; + file_path: string; + start_line: number | undefined; + end_line: number | undefined; + score: number; + content: string; +}; + export default class SnippetIndex { #insertSnippet: sqlite3.Statement; #updateSnippetBoost: sqlite3.Statement; @@ -100,7 +109,7 @@ export default class SnippetIndex { } searchSnippets(query: string, limit = 10): SnippetSearchResult[] { - const rows = this.#searchSnippet.all(query, limit) as any[]; + const rows = this.#searchSnippet.all(query, limit) as SnippetSearchRow[]; return rows.map((row) => ({ directory: row.directory, snippetId: row.snippet_id, diff --git a/packages/search/src/split-camelized.ts b/packages/search/src/split-camelized.ts index f3708d2e8b..167cd81f09 100644 --- a/packages/search/src/split-camelized.ts +++ b/packages/search/src/split-camelized.ts @@ -21,16 +21,16 @@ export function splitCamelized( // want to preserve uppercase sequences, we cannot // simply lowercase the separated string at the end. // `data_For_USACounties` → `data_for_USACounties` - decamelized = decamelized.replace( + const result = decamelized.replace( /((? $0.toLowerCase() ); // Remaining uppercase sequences will be separated from lowercase sequences. // `data_For_USACounties` → `data_for_USA_counties` - return decamelized.replace( + return result.replace( /(\p{Uppercase_Letter}+)(\p{Uppercase_Letter}\p{Lowercase_Letter}+)/gu, - (_, $1, $2) => $1 + separator + $2.toLowerCase() + (_, $1, $2: string) => $1 + separator + $2.toLowerCase() ); }; diff --git a/packages/search/src/splitter.ts b/packages/search/src/splitter.ts index 2188ba77d4..85d2665c47 100644 --- a/packages/search/src/splitter.ts +++ b/packages/search/src/splitter.ts @@ -3,7 +3,6 @@ import { SupportedTextSplitterLanguage, } from 'langchain/text_splitter'; import makeDebug from 'debug'; -import { warn } from 'console'; export type Chunk = { content: string; @@ -34,16 +33,6 @@ const TEXT_SPLITTER_LANGUAGE_EXTENSIONS: Record { - return Promise.resolve([ - { - content, - startLine: 1, - endLine: content.split('\n').length, - }, - ]); -} - export async function langchainSplitter(content: string, fileExtension: string): Promise { const language = Object.keys(TEXT_SPLITTER_LANGUAGE_EXTENSIONS).find((language) => TEXT_SPLITTER_LANGUAGE_EXTENSIONS[language as SupportedTextSplitterLanguage].includes( @@ -63,7 +52,8 @@ export async function langchainSplitter(content: string, fileExtension: string): // { loc: { lines: { from: 1, to: 14 } } } return documents.map((doc) => { - const lines = doc.metadata?.loc?.lines; + const loc = doc.metadata?.loc as { lines: { from: number; to: number } } | undefined; + const lines = loc?.lines; const result: Chunk = { content: doc.pageContent, }; From 7dccc283e9efd762ce0ed1c5e1ddb52b2b77aa5d Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 25 Oct 2024 11:22:07 -0400 Subject: [PATCH 14/15] chore: Upgrade `better-sqlite3` to v11.5.0 --- packages/cli/package.json | 2 +- yarn.lock | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index b8f823bdc5..080a0a6144 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -103,7 +103,7 @@ "applicationinsights": "^2.1.4", "async": "^3.2.4", "axios": "^0.27.2", - "better-sqlite3": "^9.5.0", + "better-sqlite3": "^11.5.0", "body-parser": "^1.20.2", "boxen": "^5.0.1", "chalk": "^4.1.2", diff --git a/yarn.lock b/yarn.lock index acf6d6e731..b49c265287 100644 --- a/yarn.lock +++ b/yarn.lock @@ -182,7 +182,7 @@ __metadata: applicationinsights: ^2.1.4 async: ^3.2.4 axios: ^0.27.2 - better-sqlite3: ^9.5.0 + better-sqlite3: ^11.5.0 body-parser: ^1.20.2 boxen: ^5.0.1 chalk: ^4.1.2 @@ -16178,17 +16178,6 @@ __metadata: languageName: node linkType: hard -"better-sqlite3@npm:^9.5.0": - version: 9.5.0 - resolution: "better-sqlite3@npm:9.5.0" - dependencies: - bindings: ^1.5.0 - node-gyp: latest - prebuild-install: ^7.1.1 - checksum: cfa56519755d6dd29ef8361c872c6f29392fcfcc44a435099eb61729aae23c4f18057972592a0b9ff0fcedc98e43db4ee7a1a2de21ab2868f8dcec4f8272ad9d - languageName: node - linkType: hard - "bfj@npm:^6.1.1": version: 6.1.2 resolution: "bfj@npm:6.1.2" From b97c3c6a5d0b1676c2f6c734be9b395f43a8c4d2 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Tue, 29 Oct 2024 12:49:55 -0400 Subject: [PATCH 15/15] fixup! feat: Refactor fulltext search to use @appland/search --- .../src/rpc/explain/SearchContextCollector.ts | 32 ++++++++--- packages/cli/src/rpc/explain/buildIndex.ts | 44 +++++++++++++++ packages/cli/src/rpc/explain/index-files.ts | 54 ++++++++++--------- 3 files changed, 100 insertions(+), 30 deletions(-) create mode 100644 packages/cli/src/rpc/explain/buildIndex.ts diff --git a/packages/cli/src/rpc/explain/SearchContextCollector.ts b/packages/cli/src/rpc/explain/SearchContextCollector.ts index 5d85182975..75f0f32eb3 100644 --- a/packages/cli/src/rpc/explain/SearchContextCollector.ts +++ b/packages/cli/src/rpc/explain/SearchContextCollector.ts @@ -3,6 +3,8 @@ import sqlite3 from 'better-sqlite3'; import { ContextV2, applyContext } from '@appland/navie'; import { SearchRpc } from '@appland/rpc'; +import { FileSearchResult } from '@appland/search'; + import AppMapIndex, { SearchResponse as AppMapSearchResponse, SearchOptions as AppMapSearchOptions, @@ -12,6 +14,7 @@ import EventCollector from './EventCollector'; import indexFiles from './index-files'; import indexSnippets from './index-snippets'; import collectSnippets from './collect-snippets'; +import buildIndex from './buildIndex'; export default class SearchContextCollector { public excludePatterns: RegExp[] | undefined; @@ -67,9 +70,27 @@ export default class SearchContextCollector { ); } - const db = new sqlite3(':memory:'); - const fileIndex = await indexFiles(db, this.sourceDirectories); - const fileSearchResults = fileIndex.search(this.vectorTerms.join(' OR ')); + const fileIndex = await buildIndex('files', async (indexFile) => { + const db = new sqlite3(indexFile); + return await indexFiles( + db, + this.sourceDirectories, + this.includePatterns, + this.excludePatterns + ); + }); + let fileSearchResults: FileSearchResult[]; + try { + fileSearchResults = fileIndex.index.search(this.vectorTerms.join(' OR ')); + } finally { + fileIndex.close(); + } + + const snippetIndex = await buildIndex('snippets', async (indexFile) => { + const db = new sqlite3(indexFile); + return await indexSnippets(db, fileSearchResults); + }); + let contextCandidate: { results: SearchRpc.SearchResult[]; context: ContextV2.ContextResponse; @@ -77,7 +98,6 @@ export default class SearchContextCollector { }; try { const eventsCollector = new EventCollector(this.vectorTerms.join(' '), appmapSearchResponse); - const snippetIndex = await indexSnippets(db, fileSearchResults); let charCount = 0; let maxEventsPerDiagram = 5; @@ -98,7 +118,7 @@ export default class SearchContextCollector { const charLimit = codeSnippetCount === 0 ? this.charLimit : this.charLimit / 4; const sourceContext = collectSnippets( - snippetIndex, + snippetIndex.index, this.vectorTerms.join(' OR '), charLimit ); @@ -121,7 +141,7 @@ export default class SearchContextCollector { log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`); } } finally { - db.close(); + snippetIndex.close(); } return { diff --git a/packages/cli/src/rpc/explain/buildIndex.ts b/packages/cli/src/rpc/explain/buildIndex.ts new file mode 100644 index 0000000000..61e78d05f5 --- /dev/null +++ b/packages/cli/src/rpc/explain/buildIndex.ts @@ -0,0 +1,44 @@ +// This function is used for a "one-shot" search in which the index us built, used, and discarded. +// Replace this with a persistent index file that can be used across multiple searches, and is + +import { rmSync } from 'fs'; +import { mkdtemp } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +export interface Closeable { + close(): void; +} + +type CloseableIndex = { + index: T; + close: () => void; +}; + +export default async function buildIndex( + indexName: string, + builder: (indexFile: string) => Promise +): Promise> { + const tmpDir = await mkdtemp(join(tmpdir(), `appmap-${indexName}-${new Date().getTime()}`)); + const indexFile = join(tmpDir, 'index.sqlite'); + + const index = await builder(indexFile); + + const close = () => { + try { + index.close(); + } catch (err) { + console.error(err); + } + try { + rmSync(tmpDir, { recursive: true }); + } catch (err) { + console.error(err); + } + }; + + return { + index, + close, + }; +} diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts index d4646d8113..d49d4b09c5 100644 --- a/packages/cli/src/rpc/explain/index-files.ts +++ b/packages/cli/src/rpc/explain/index-files.ts @@ -5,45 +5,51 @@ import { buildFileIndex, FileIndex, fileTokens, + FilterFn, isBinaryFile, isDataFile, isLargeFile, listProjectFiles, readFileSafe, } from '@appland/search'; +import { fileNameMatchesFilterPatterns } from '../../fulltext/fileNameMatchesFilterPatterns'; const debug = makeDebug('appmap:rpc:explain:index-files'); -const fileFilter = async (path: string) => { - debug('Filtering: %s', path); - if (isBinaryFile(path)) { - debug('Skipping binary file: %s', path); - return false; - } - - const isData = isDataFile(path); - if (isData && (await isLargeFile(path))) { - debug('Skipping large data file: %s', path); - return false; - } - - return true; -}; +function fileFilter( + includePatterns: RegExp[] | undefined, + excludePatterns: RegExp[] | undefined +): FilterFn { + return async (path: string) => { + debug('Filtering: %s', path); + if (isBinaryFile(path)) { + debug('Skipping binary file: %s', path); + return false; + } + + const includeFile = fileNameMatchesFilterPatterns(path, includePatterns, excludePatterns); + if (!includeFile) return false; + + const isData = isDataFile(path); + if (isData && (await isLargeFile(path))) { + debug('Skipping large data file: %s', path); + return false; + } + + return true; + }; +} export default async function indexFiles( db: sqlite3.Database, - directories: string[] + directories: string[], + includePatterns: RegExp[] | undefined, + excludePatterns: RegExp[] | undefined ): Promise { const fileIndex = new FileIndex(db); - await buildFileIndex( - fileIndex, - directories, - listProjectFiles, - fileFilter, - readFileSafe, - fileTokens - ); + const filter = fileFilter(includePatterns, excludePatterns); + await buildFileIndex(fileIndex, directories, listProjectFiles, filter, readFileSafe, fileTokens); return fileIndex; }