From ba282040898b0bbf2aa4768e634494c3a74af78a Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Tue, 22 Aug 2023 17:03:02 -0400 Subject: [PATCH 01/18] temp: only run problematic test in CI for now --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b28ba07b..be16ad1d 100644 --- a/package.json +++ b/package.json @@ -425,7 +425,7 @@ "test:system": "ts-node test/systemTest.ts", "test:unit": "mocha test/unit/**/*.test.[tj]s", "test:unit:some": "mocha", - "test": "yarn test:unit && yarn test:web-client && yarn test:integration && yarn test:system", + "test": "yarn test:integration scanner/performAsAppMapsAreModified.test.js", "compile": "NODE_ENV=production tsup --config tsup.config.ts", "watch": "tsup --config tsup.config.ts --watch", "package": "vsce package", From a8ac0ab598d9647d6bfaca185d55de7979b05ca7 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 16:34:52 -0400 Subject: [PATCH 02/18] test: disable keytar to avoid encryption error --- test/integrationTest.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integrationTest.ts b/test/integrationTest.ts index 83733996..ea979840 100644 --- a/test/integrationTest.ts +++ b/test/integrationTest.ts @@ -142,6 +142,8 @@ async function integrationTest() { userDataDir, // '--disable-extensions', '--disable-gpu', + '--disable-keytar', + '--password-store=basic', workspaceDir, ], }); From 5e7d554ffd2e61cf8d9399fc708241994c959bfb Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 16:35:40 -0400 Subject: [PATCH 03/18] test: get output from executeWorkspaceOSCommand --- test/integration/util.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/integration/util.ts b/test/integration/util.ts index f9e51eff..684cf3dc 100644 --- a/test/integration/util.ts +++ b/test/integration/util.ts @@ -237,15 +237,18 @@ async function closeAllEditors(): Promise { await vscode.commands.executeCommand('workbench.action.closeAllEditors'); } -export async function executeWorkspaceOSCommand(cmd: string, workspaceName: string): Promise { - return new Promise((resolve, reject) => { +export async function executeWorkspaceOSCommand( + cmd: string, + workspaceName: string +): Promise { + return new Promise((resolve, reject) => { exec(cmd, { cwd: workspaceName }, (err, stdout, stderr) => { if (err) { console.log(stdout); console.warn(stderr); return reject(err); } - resolve(); + resolve(stdout); }); }); } From 768455dca6f519f0139c3d584720464782998a01 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 16:45:11 -0400 Subject: [PATCH 04/18] temp --- .../performAsAppMapsAreModified.test.ts | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/test/integration/scanner/performAsAppMapsAreModified.test.ts b/test/integration/scanner/performAsAppMapsAreModified.test.ts index cb716a61..b4db5c00 100644 --- a/test/integration/scanner/performAsAppMapsAreModified.test.ts +++ b/test/integration/scanner/performAsAppMapsAreModified.test.ts @@ -1,8 +1,7 @@ import * as vscode from 'vscode'; import assert from 'assert'; -import { exists, stat } from 'fs'; +import { existsSync } from 'fs'; import { join, relative } from 'path'; -import { promisify } from 'util'; import { initializeWorkspace, waitFor, @@ -15,8 +14,19 @@ import { waitForAppMapServices, restoreFile, withAuthenticatedUser, + executeWorkspaceOSCommand, } from '../util'; +// async function logWatches(): Promise { +// const watches = await executeWorkspaceOSCommand('ps -ef | grep -e --watch', ProjectA); +// console.log('Watches: ', watches); +// } + +async function logIndexDir(): Promise { + const indexDir = await executeWorkspaceOSCommand(`ls -al ${ExampleAppMapIndexDir}`, ProjectA); + console.log('Index Dir: ', indexDir); +} + describe('Scanner', () => { withAuthenticatedUser(); @@ -25,9 +35,7 @@ describe('Scanner', () => { await waitForAppMapServices( 'tmp/appmap/minitest/Microposts_controller_can_get_microposts_as_JSON.appmap.json' ); - await waitFor('Index directory exists', async () => { - return Boolean(await promisify(stat)(ExampleAppMapIndexDir)); - }); + await waitFor('Index directory exists', () => existsSync(ExampleAppMapIndexDir)); }); afterEach(initializeWorkspace); @@ -40,23 +48,31 @@ describe('Scanner', () => { // KEG: Remove this, as it's failing due to an issue either with IndexJanitor or with the test itself. // await waitFor( // `AppMap index dir should be removed`, - // async () => !(await promisify(exists)(ExampleAppMapIndexDir)) + // async () => !existsSync(ExampleAppMapIndexDir) // ); + await logIndexDir(); const appMapPath = relative(ProjectA, ExampleAppMap); - await repeatUntil( - async () => await restoreFile(appMapPath), - `AppMap should be reindexed`, - async () => promisify(exists)(join(ExampleAppMapIndexDir, 'mtime')) + await restoreFile(appMapPath); + + assert(existsSync(ExampleAppMap)); + await logIndexDir(); + + await waitFor('AppMap to be re-indexed', () => + existsSync(join(ExampleAppMapIndexDir, 'mtime')) + ); + await logIndexDir(); + + await waitFor('Findings to be generated', () => + existsSync(join(ExampleAppMapIndexDir, 'appmap-findings.json')) ); + await logIndexDir(); - await repeatUntil( - async () => await restoreFile(appMapPath), - 'No Diagnostics were created for Microposts_controller_can_get_microposts_as_JSON', - async () => { - return getDiagnosticsForAppMap(ExampleAppMap).length > 0; - } + await waitFor( + 'diagnostics to be created', + () => getDiagnosticsForAppMap(ExampleAppMap).length > 0 ); + await logIndexDir(); const diagnostic = getDiagnosticsForAppMap(ExampleAppMap)[0]; assert.strictEqual( From e92d95fd241b6253cc84a35b1a50addbdc4c960b Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 17:07:19 -0400 Subject: [PATCH 05/18] temp --- .../performAsAppMapsAreModified.test.ts | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test/integration/scanner/performAsAppMapsAreModified.test.ts b/test/integration/scanner/performAsAppMapsAreModified.test.ts index b4db5c00..972645a7 100644 --- a/test/integration/scanner/performAsAppMapsAreModified.test.ts +++ b/test/integration/scanner/performAsAppMapsAreModified.test.ts @@ -16,6 +16,7 @@ import { withAuthenticatedUser, executeWorkspaceOSCommand, } from '../util'; +import AppMapService from '../../../src/appMapService'; // async function logWatches(): Promise { // const watches = await executeWorkspaceOSCommand('ps -ef | grep -e --watch', ProjectA); @@ -28,11 +29,12 @@ async function logIndexDir(): Promise { } describe('Scanner', () => { + let services: AppMapService; withAuthenticatedUser(); beforeEach(async () => { await initializeWorkspace(); - await waitForAppMapServices( + services = await waitForAppMapServices( 'tmp/appmap/minitest/Microposts_controller_can_get_microposts_as_JSON.appmap.json' ); await waitFor('Index directory exists', () => existsSync(ExampleAppMapIndexDir)); @@ -41,29 +43,28 @@ describe('Scanner', () => { afterEach(initializeWorkspace); it('is performed as AppMaps are modified', async () => { - await vscode.commands.executeCommand('appmap.deleteAllAppMaps'); + async function removeAndReindex() { + await vscode.commands.executeCommand('appmap.deleteAllAppMaps'); - await waitFor('Diagnostics were not cleared', hasNoDiagnostics); + await waitFor('AppMaps to be deleted', () => services.localAppMaps.appMaps.length === 0); + await waitFor('Diagnostics to be cleared', hasNoDiagnostics); - // KEG: Remove this, as it's failing due to an issue either with IndexJanitor or with the test itself. - // await waitFor( - // `AppMap index dir should be removed`, - // async () => !existsSync(ExampleAppMapIndexDir) - // ); + await logIndexDir(); + const appMapPath = relative(ProjectA, ExampleAppMap); + await restoreFile(appMapPath); - await logIndexDir(); - const appMapPath = relative(ProjectA, ExampleAppMap); - await restoreFile(appMapPath); + assert(existsSync(ExampleAppMap)); + await logIndexDir(); - assert(existsSync(ExampleAppMap)); - await logIndexDir(); + await waitFor('AppMap to be re-indexed', () => + existsSync(join(ExampleAppMapIndexDir, 'mtime')) + ); + await logIndexDir(); + } - await waitFor('AppMap to be re-indexed', () => - existsSync(join(ExampleAppMapIndexDir, 'mtime')) - ); - await logIndexDir(); + await removeAndReindex(); - await waitFor('Findings to be generated', () => + await repeatUntil(removeAndReindex, 'Findings to be generated', () => existsSync(join(ExampleAppMapIndexDir, 'appmap-findings.json')) ); await logIndexDir(); From 57d015b609e4321fb05c76e69fce5cbc1ff57bc2 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 17:18:39 -0400 Subject: [PATCH 06/18] temp --- .../scanner/performAsAppMapsAreModified.test.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/integration/scanner/performAsAppMapsAreModified.test.ts b/test/integration/scanner/performAsAppMapsAreModified.test.ts index 972645a7..7827a79e 100644 --- a/test/integration/scanner/performAsAppMapsAreModified.test.ts +++ b/test/integration/scanner/performAsAppMapsAreModified.test.ts @@ -1,6 +1,6 @@ import * as vscode from 'vscode'; import assert from 'assert'; -import { existsSync } from 'fs'; +import { existsSync, rmSync } from 'fs'; import { join, relative } from 'path'; import { initializeWorkspace, @@ -23,10 +23,10 @@ import AppMapService from '../../../src/appMapService'; // console.log('Watches: ', watches); // } -async function logIndexDir(): Promise { - const indexDir = await executeWorkspaceOSCommand(`ls -al ${ExampleAppMapIndexDir}`, ProjectA); - console.log('Index Dir: ', indexDir); -} +// async function logIndexDir(): Promise { +// const indexDir = await executeWorkspaceOSCommand(`ls -al ${ExampleAppMapIndexDir}`, ProjectA); +// console.log(`\nIndex Dir: ${indexDir}`); +// } describe('Scanner', () => { let services: AppMapService; @@ -45,21 +45,19 @@ describe('Scanner', () => { it('is performed as AppMaps are modified', async () => { async function removeAndReindex() { await vscode.commands.executeCommand('appmap.deleteAllAppMaps'); + rmSync(ExampleAppMapIndexDir, { force: true, recursive: true }); await waitFor('AppMaps to be deleted', () => services.localAppMaps.appMaps.length === 0); await waitFor('Diagnostics to be cleared', hasNoDiagnostics); - await logIndexDir(); const appMapPath = relative(ProjectA, ExampleAppMap); await restoreFile(appMapPath); assert(existsSync(ExampleAppMap)); - await logIndexDir(); await waitFor('AppMap to be re-indexed', () => existsSync(join(ExampleAppMapIndexDir, 'mtime')) ); - await logIndexDir(); } await removeAndReindex(); @@ -67,13 +65,11 @@ describe('Scanner', () => { await repeatUntil(removeAndReindex, 'Findings to be generated', () => existsSync(join(ExampleAppMapIndexDir, 'appmap-findings.json')) ); - await logIndexDir(); await waitFor( 'diagnostics to be created', () => getDiagnosticsForAppMap(ExampleAppMap).length > 0 ); - await logIndexDir(); const diagnostic = getDiagnosticsForAppMap(ExampleAppMap)[0]; assert.strictEqual( From 0a4add4c055deffee18cc2dc397236637894a157 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 18:45:46 -0400 Subject: [PATCH 07/18] use 1.81.0 of scanner with verbose logging --- src/services/nodeProcessService.ts | 2 +- src/services/processWatcher.ts | 1 + src/services/scanProcessWatcher.ts | 2 +- .../scanner/performAsAppMapsAreModified.test.ts | 17 ++++++++++++----- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/services/nodeProcessService.ts b/src/services/nodeProcessService.ts index 5d05b307..6c061c0e 100644 --- a/src/services/nodeProcessService.ts +++ b/src/services/nodeProcessService.ts @@ -209,7 +209,7 @@ export class NodeProcessService implements WorkspaceService { -// const indexDir = await executeWorkspaceOSCommand(`ls -al ${ExampleAppMapIndexDir}`, ProjectA); -// console.log(`\nIndex Dir: ${indexDir}`); -// } +async function logIndexDir(): Promise { + const indexDir = await executeWorkspaceOSCommand(`ls -al ${ExampleAppMapIndexDir}`, ProjectA); + console.log(`\nIndex Dir: ${indexDir}`); +} describe('Scanner', () => { let services: AppMapService; @@ -44,8 +44,11 @@ describe('Scanner', () => { it('is performed as AppMaps are modified', async () => { async function removeAndReindex() { + await logIndexDir(); await vscode.commands.executeCommand('appmap.deleteAllAppMaps'); rmSync(ExampleAppMapIndexDir, { force: true, recursive: true }); + mkdirSync(ExampleAppMapIndexDir); + await logIndexDir(); await waitFor('AppMaps to be deleted', () => services.localAppMaps.appMaps.length === 0); await waitFor('Diagnostics to be cleared', hasNoDiagnostics); @@ -58,13 +61,16 @@ describe('Scanner', () => { await waitFor('AppMap to be re-indexed', () => existsSync(join(ExampleAppMapIndexDir, 'mtime')) ); + await logIndexDir(); } await removeAndReindex(); + await logIndexDir(); await repeatUntil(removeAndReindex, 'Findings to be generated', () => existsSync(join(ExampleAppMapIndexDir, 'appmap-findings.json')) ); + await logIndexDir(); await waitFor( 'diagnostics to be created', @@ -76,5 +82,6 @@ describe('Scanner', () => { diagnostic.uri.fsPath, join(ProjectA, 'app/controllers/microposts_controller.rb') ); + assert(false); }); }); From fd1072d0fb5d92187f836f8d77bb8125258793d2 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Wed, 23 Aug 2023 18:51:18 -0400 Subject: [PATCH 08/18] use scanner 1.77.0 --- src/services/nodeProcessService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/nodeProcessService.ts b/src/services/nodeProcessService.ts index 6c061c0e..a049cb0a 100644 --- a/src/services/nodeProcessService.ts +++ b/src/services/nodeProcessService.ts @@ -209,7 +209,7 @@ export class NodeProcessService implements WorkspaceService Date: Wed, 23 Aug 2023 19:19:52 -0400 Subject: [PATCH 09/18] scanner 1.81.1 --- src/services/nodeProcessService.ts | 2 +- test/integration/scanner/performAsAppMapsAreModified.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/nodeProcessService.ts b/src/services/nodeProcessService.ts index a049cb0a..104492b5 100644 --- a/src/services/nodeProcessService.ts +++ b/src/services/nodeProcessService.ts @@ -209,7 +209,7 @@ export class NodeProcessService implements WorkspaceService { diagnostic.uri.fsPath, join(ProjectA, 'app/controllers/microposts_controller.rb') ); - assert(false); }); }); From 6e46ec0fb5444497803ddf2201afc0f84ae11c67 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Thu, 24 Aug 2023 15:01:01 -0400 Subject: [PATCH 10/18] appmap-js as gitmodule --- .gitmodules | 3 +++ appmap-js | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 appmap-js diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e38c6fc6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "appmap-js"] + path = appmap-js + url = git@github.com:getappmap/appmap-js.git diff --git a/appmap-js b/appmap-js new file mode 160000 index 00000000..6287dbe1 --- /dev/null +++ b/appmap-js @@ -0,0 +1 @@ +Subproject commit 6287dbe1eb836342ca2c388db62566aa00b379b1 From 9d4a35f60eada3b4a6342a778154757dc2fb24b5 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Thu, 24 Aug 2023 15:01:19 -0400 Subject: [PATCH 11/18] use gitmodule for scanner --- src/services/nodeDependencyProcess.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/services/nodeDependencyProcess.ts b/src/services/nodeDependencyProcess.ts index d836d794..a26cda42 100644 --- a/src/services/nodeDependencyProcess.ts +++ b/src/services/nodeDependencyProcess.ts @@ -111,6 +111,15 @@ export class ProcessLog extends Array { } export async function getModulePath(options: ProgramOptions): Promise { + const workspaceFolders = vscode.workspace.workspaceFolders; + if ( + options.dependency === ProgramName.Scanner && + workspaceFolders && + workspaceFolders.length > 0 + ) { + const root = path.resolve(workspaceFolders[0].uri.fsPath, '..', '..', '..', '..'); + return path.join(root, 'appmap-js', 'packages', 'scanner', 'built', 'cli.js'); + } const localToolsPath = ExtensionSettings.appMapCommandLineToolsPath; if (localToolsPath) { let packageName: string; From 76860e016e162287a3fa9078a9c2ca28e677192b Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Thu, 24 Aug 2023 15:01:32 -0400 Subject: [PATCH 12/18] fail test to get stdout from scanner --- test/integration/scanner/performAsAppMapsAreModified.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/scanner/performAsAppMapsAreModified.test.ts b/test/integration/scanner/performAsAppMapsAreModified.test.ts index aaa609fe..fe69863e 100644 --- a/test/integration/scanner/performAsAppMapsAreModified.test.ts +++ b/test/integration/scanner/performAsAppMapsAreModified.test.ts @@ -82,5 +82,6 @@ describe('Scanner', () => { diagnostic.uri.fsPath, join(ProjectA, 'app/controllers/microposts_controller.rb') ); + assert(false); }); }); From 7235948b827e2751e0fbe71d357067782e25b54a Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Thu, 24 Aug 2023 15:02:28 -0400 Subject: [PATCH 13/18] stop verbose logging for scanner --- src/services/scanProcessWatcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/scanProcessWatcher.ts b/src/services/scanProcessWatcher.ts index 67198112..1394beff 100644 --- a/src/services/scanProcessWatcher.ts +++ b/src/services/scanProcessWatcher.ts @@ -14,7 +14,7 @@ export default class ScanProcessWatcher extends ProcessWatcher { id: ProcessId.Analysis, modulePath: modulePath, log: NodeProcessService.outputChannel, - args: ['scan', '--watch', '--appmap-dir', appmapDir, '--verbose'], + args: ['scan', '--watch', '--appmap-dir', appmapDir], cwd, env, }); From fa7bb67f7e323cc4c16b5c8ff25f08425e4c7eec Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Fri, 25 Aug 2023 11:54:50 -0400 Subject: [PATCH 14/18] switch gitmodule from ssh to https --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e38c6fc6..3fd54d05 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "appmap-js"] path = appmap-js - url = git@github.com:getappmap/appmap-js.git + url = https://github.com/getappmap/appmap-js.git From e1c1adc39e14d0fbb17fb72b6dc79848329b9375 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Fri, 25 Aug 2023 12:13:33 -0400 Subject: [PATCH 15/18] update appmap-js submodule --- appmap-js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appmap-js b/appmap-js index 6287dbe1..fd5b509f 160000 --- a/appmap-js +++ b/appmap-js @@ -1 +1 @@ -Subproject commit 6287dbe1eb836342ca2c388db62566aa00b379b1 +Subproject commit fd5b509f337737859a32b0388e13fdd82e3050cb From ec932e585fe8c1792c17cac956feff47116ae56f Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Fri, 25 Aug 2023 12:21:01 -0400 Subject: [PATCH 16/18] skip linting --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index be16ad1d..cae95685 100644 --- a/package.json +++ b/package.json @@ -419,7 +419,7 @@ "scripts": { "vscode:prepublish": "yarn run compile", "lint": "eslint . --ext .ts", - "pretest": "yarn run lint && tsc --strict --skipLibCheck", + "pretest": "tsc --strict --skipLibCheck", "test:integration": "ts-node ./test/integrationTest.ts", "test:web-client": "mocha web/test/*.test.mjs", "test:system": "ts-node test/systemTest.ts", From 962660e422ce39b4116eb633907839b513456593 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Fri, 25 Aug 2023 12:34:27 -0400 Subject: [PATCH 17/18] update appmap-js submodue with node_modules --- appmap-js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appmap-js b/appmap-js index fd5b509f..6ece1d52 160000 --- a/appmap-js +++ b/appmap-js @@ -1 +1 @@ -Subproject commit fd5b509f337737859a32b0388e13fdd82e3050cb +Subproject commit 6ece1d5279ef20fa64a27016548a16f503fc65e9 From 170d73d39d97cdd59bb20f6b2f652b484ddf5c53 Mon Sep 17 00:00:00 2001 From: Adam Trotta Date: Fri, 25 Aug 2023 13:06:57 -0400 Subject: [PATCH 18/18] do not use submodules --- .gitmodules | 3 --- appmap-js | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 appmap-js diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 3fd54d05..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "appmap-js"] - path = appmap-js - url = https://github.com/getappmap/appmap-js.git diff --git a/appmap-js b/appmap-js deleted file mode 160000 index 6ece1d52..00000000 --- a/appmap-js +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6ece1d5279ef20fa64a27016548a16f503fc65e9