From bca7ecb7820b6c292170b43a3142f530498bf10f Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 13 Nov 2023 11:13:36 +0100 Subject: [PATCH 01/11] Add source to database item This adds a source property to the database item to store the source of the database, for example GitHub or an internet URL. This will be used to automatically check for updates to GitHub-downloaded databases in the future. --- .../src/databases/config/db-config.ts | 3 ++ .../src/databases/database-fetcher.ts | 28 ++++++++++++++++- extensions/ql-vscode/src/databases/db-item.ts | 3 ++ .../src/databases/db-tree-creator.ts | 1 + .../src/databases/local-databases-ui.ts | 11 +++++-- .../local-databases/database-item-impl.ts | 5 ++++ .../local-databases/database-item.ts | 6 ++++ .../local-databases/database-manager.ts | 16 ++++++++-- .../local-databases/database-options.ts | 4 +++ .../local-databases/database-source.ts | 30 +++++++++++++++++++ .../ql-vscode/src/debugger/debugger-ui.ts | 4 ++- .../src/query-testing/test-runner.ts | 1 + .../test/factories/databases/databases.ts | 3 ++ .../test/factories/db-config-factories.ts | 6 ++++ .../test/factories/db-item-factories.ts | 6 ++++ .../databases/db-item-selection.test.ts | 3 ++ .../databases/db-tree-creator.test.ts | 19 ++++++++++++ .../databases/db-panel-rendering.test.ts | 30 +++++++++++++++++++ .../local-queries/local-databases.test.ts | 29 ++++++++++++++---- .../query-testing/test-runner.test.ts | 11 +++++-- 20 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-source.ts diff --git a/extensions/ql-vscode/src/databases/config/db-config.ts b/extensions/ql-vscode/src/databases/config/db-config.ts index 2c827c7086c..7825f17e92c 100644 --- a/extensions/ql-vscode/src/databases/config/db-config.ts +++ b/extensions/ql-vscode/src/databases/config/db-config.ts @@ -1,6 +1,8 @@ // Contains models and consts for the data we want to store in the database config. // Changes to these models should be done carefully and account for backwards compatibility of data. +import { DatabaseSource } from "../local-databases/database-source"; + export const DB_CONFIG_VERSION = 1; export interface DbConfig { @@ -88,6 +90,7 @@ export interface LocalDatabase { name: string; dateAdded: number; language: string; + source: DatabaseSource; storagePath: string; } diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index 5521dcd00c3..3e2f5106738 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -33,6 +33,7 @@ import { allowHttp } from "../config"; import { showAndLogInformationMessage } from "../common/logging"; import { AppOctokit } from "../common/octokit"; import { getLanguageDisplayName } from "../common/query-language"; +import { DatabaseSource } from "./local-databases/database-source"; /** * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file. @@ -62,6 +63,10 @@ export async function promptImportInternetDatabase( databaseManager, storagePath, undefined, + { + type: "url", + url: databaseUrl, + }, progress, cli, ); @@ -199,7 +204,7 @@ export async function downloadGitHubDatabase( return; } - const { databaseUrl, name, owner } = result; + const { databaseUrl, name, owner, commitOid } = result; /** * The 'token' property of the token object returned by `octokit.auth()`. @@ -221,6 +226,11 @@ export async function downloadGitHubDatabase( databaseManager, storagePath, `${owner}/${name}`, + { + type: "github", + repository: nwo, + commitOid, + }, progress, cli, makeSelected, @@ -250,6 +260,10 @@ export async function importArchiveDatabase( databaseManager, storagePath, undefined, + { + type: "archive", + path: databaseUrl, + }, progress, cli, ); @@ -282,6 +296,7 @@ export async function importArchiveDatabase( * @param databaseManager the DatabaseManager * @param storagePath where to store the unzipped database. * @param nameOverride a name for the database that overrides the default + * @param source the source of the database * @param progress callback to send progress messages to * @param makeSelected make the new database selected in the databases panel (default: true) * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace @@ -292,6 +307,7 @@ async function databaseArchiveFetcher( databaseManager: DatabaseManager, storagePath: string, nameOverride: string | undefined, + source: DatabaseSource, progress: ProgressCallback, cli?: CodeQLCliServer, makeSelected = true, @@ -336,6 +352,7 @@ async function databaseArchiveFetcher( const item = await databaseManager.openDatabase( Uri.file(dbPath), + source, makeSelected, nameOverride, { @@ -533,6 +550,7 @@ export async function convertGithubNwoToDatabaseUrl( databaseUrl: string; owner: string; name: string; + commitOid: string | null; } | undefined > { @@ -553,10 +571,18 @@ export async function convertGithubNwoToDatabaseUrl( } } + const databaseForLanguage = response.data.find( + (db: any) => db.language === language, + ); + if (!databaseForLanguage) { + throw new Error(`No database found for language '${language}'`); + } + return { databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`, owner, name: repo, + commitOid: databaseForLanguage.commit_oid, }; } catch (e) { void extLogger.log(`Error: ${getErrorMessage(e)}`); diff --git a/extensions/ql-vscode/src/databases/db-item.ts b/extensions/ql-vscode/src/databases/db-item.ts index 70f380c911e..d2c98717526 100644 --- a/extensions/ql-vscode/src/databases/db-item.ts +++ b/extensions/ql-vscode/src/databases/db-item.ts @@ -1,5 +1,7 @@ // This file contains models that are used to represent the databases. +import { DatabaseSource } from "./local-databases/database-source"; + export enum DbItemKind { RootLocal = "RootLocal", LocalList = "LocalList", @@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem { databaseName: string; dateAdded: number; language: string; + source: DatabaseSource; storagePath: string; parentListName?: string; } diff --git a/extensions/ql-vscode/src/databases/db-tree-creator.ts b/extensions/ql-vscode/src/databases/db-tree-creator.ts index f5ecda98c12..a804603c895 100644 --- a/extensions/ql-vscode/src/databases/db-tree-creator.ts +++ b/extensions/ql-vscode/src/databases/db-tree-creator.ts @@ -197,6 +197,7 @@ function createLocalDb( databaseName: db.name, dateAdded: db.dateAdded, language: db.language, + source: db.source, storagePath: db.storagePath, selected: !!selected, parentListName: listName, diff --git a/extensions/ql-vscode/src/databases/local-databases-ui.ts b/extensions/ql-vscode/src/databases/local-databases-ui.ts index bf6b7d59f2f..6e66a5af55d 100644 --- a/extensions/ql-vscode/src/databases/local-databases-ui.ts +++ b/extensions/ql-vscode/src/databases/local-databases-ui.ts @@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject { await this.databaseManager.openDatabase( uri, + { + type: "folder", + }, makeSelected, nameOverride, { @@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject { this.queryServer?.cliServer, ); } else { - await this.databaseManager.openDatabase(uri); + await this.databaseManager.openDatabase(uri, { + type: "folder", + }); } } catch (e) { // rethrow and let this be handled by default error handling. @@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject { if (byFolder) { const fixedUri = await this.fixDbUri(uri); // we are selecting a database folder - return await this.databaseManager.openDatabase(fixedUri); + return await this.databaseManager.openDatabase(fixedUri, { + type: "folder", + }); } else { // we are selecting a database archive. Must unzip into a workspace-controlled area // before importing. diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts index c802f65ea84..76d98259423 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics"; import { stat } from "fs-extra"; import { containsPath, pathsEqual } from "../../common/files"; import { DatabaseContents } from "./database-contents"; +import { DatabaseSource } from "./database-source"; export class DatabaseItemImpl implements DatabaseItem { // These are only public in the implementation, they are readonly in the interface @@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem { return this.options.dateAdded; } + public get source(): DatabaseSource | undefined { + return this.options.source; + } + public resolveSourceFile(uriStr: string | undefined): vscode.Uri { const sourceArchive = this.sourceArchive; const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts index ad64f52d43b..e9774a4d4f0 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts @@ -2,6 +2,7 @@ import vscode from "vscode"; import * as cli from "../../codeql-cli/cli"; import { DatabaseContents } from "./database-contents"; import { DatabaseOptions } from "./database-options"; +import { DatabaseSource } from "./database-source"; /** An item in the list of available databases */ export interface DatabaseItem { @@ -25,6 +26,11 @@ export interface DatabaseItem { */ readonly dateAdded: number | undefined; + /** + * The source this database item was added from or undefined if unknown. + */ + readonly source: DatabaseSource | undefined; + /** If the database is invalid, describes why. */ readonly error: Error | undefined; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index aeecc96e016..7cfc5ef409e 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -34,6 +34,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; import { DatabaseResolver } from "./database-resolver"; import { telemetryListener } from "../../common/vscode/telemetry"; import { LanguageContextStore } from "../../language-context-store"; +import { DatabaseSource } from "./database-source"; /** * The name of the key in the workspaceState dictionary in which we @@ -131,6 +132,7 @@ export class DatabaseManager extends DisposableObject { */ public async openDatabase( uri: vscode.Uri, + source: DatabaseSource | undefined, makeSelected = true, displayName?: string, { @@ -138,7 +140,11 @@ export class DatabaseManager extends DisposableObject { addSourceArchiveFolder = true, }: OpenDatabaseOptions = {}, ): Promise { - const databaseItem = await this.createDatabaseItem(uri, displayName); + const databaseItem = await this.createDatabaseItem( + uri, + source, + displayName, + ); return await this.addExistingDatabaseItem( databaseItem, @@ -189,6 +195,7 @@ export class DatabaseManager extends DisposableObject { */ private async createDatabaseItem( uri: vscode.Uri, + source: DatabaseSource | undefined, displayName: string | undefined, ): Promise { const contents = await DatabaseResolver.resolveDatabaseContents(uri); @@ -197,6 +204,7 @@ export class DatabaseManager extends DisposableObject { displayName, dateAdded: Date.now(), language: await this.getPrimaryLanguage(uri.fsPath), + source, }; const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions); @@ -212,6 +220,7 @@ export class DatabaseManager extends DisposableObject { */ public async createOrOpenDatabaseItem( uri: vscode.Uri, + source: DatabaseSource | undefined, ): Promise { const existingItem = this.findDatabaseItem(uri); if (existingItem !== undefined) { @@ -220,7 +229,7 @@ export class DatabaseManager extends DisposableObject { } // We don't add this to the list automatically, but the user can add it later. - return this.createDatabaseItem(uri, undefined); + return this.createDatabaseItem(uri, source, undefined); } public async createSkeletonPacks(databaseItem: DatabaseItem) { @@ -355,6 +364,7 @@ export class DatabaseManager extends DisposableObject { let displayName: string | undefined = undefined; let dateAdded = undefined; let language = undefined; + let source = undefined; if (state.options) { if (typeof state.options.displayName === "string") { displayName = state.options.displayName; @@ -363,6 +373,7 @@ export class DatabaseManager extends DisposableObject { dateAdded = state.options.dateAdded; } language = state.options.language; + source = state.options.source; } const dbBaseUri = vscode.Uri.parse(state.uri, true); @@ -375,6 +386,7 @@ export class DatabaseManager extends DisposableObject { displayName, dateAdded, language, + source, }; const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions); diff --git a/extensions/ql-vscode/src/databases/local-databases/database-options.ts b/extensions/ql-vscode/src/databases/local-databases/database-options.ts index 5ee07e1c117..2dc97e1e4e0 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-options.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-options.ts @@ -1,10 +1,14 @@ +import { DatabaseSource } from "./database-source"; + export interface DatabaseOptions { displayName?: string; dateAdded?: number | undefined; language?: string; + source?: DatabaseSource; } export interface FullDatabaseOptions extends DatabaseOptions { dateAdded: number | undefined; language: string | undefined; + source: DatabaseSource | undefined; } diff --git a/extensions/ql-vscode/src/databases/local-databases/database-source.ts b/extensions/ql-vscode/src/databases/local-databases/database-source.ts new file mode 100644 index 00000000000..9afbafafabf --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-source.ts @@ -0,0 +1,30 @@ +interface DatabaseSourceFolder { + type: "folder"; +} + +interface DatabaseSourceArchive { + type: "archive"; + path: string; +} + +interface DatabaseSourceGitHub { + type: "github"; + repository: string; + commitOid: string | null; +} + +interface DatabaseSourceInternet { + type: "url"; + url: string; +} + +interface DatabaseSourceDebugger { + type: "debugger"; +} + +export type DatabaseSource = + | DatabaseSourceFolder + | DatabaseSourceArchive + | DatabaseSourceGitHub + | DatabaseSourceInternet + | DatabaseSourceDebugger; diff --git a/extensions/ql-vscode/src/debugger/debugger-ui.ts b/extensions/ql-vscode/src/debugger/debugger-ui.ts index 26d053f2713..3af1b45ea2d 100644 --- a/extensions/ql-vscode/src/debugger/debugger-ui.ts +++ b/extensions/ql-vscode/src/debugger/debugger-ui.ts @@ -105,7 +105,9 @@ class QLDebugAdapterTracker body: CodeQLProtocol.EvaluationStartedEvent["body"], ): Promise { const dbUri = Uri.file(this.configuration.database); - const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri); + const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri, { + type: "debugger", + }); // When cancellation is requested from the query history view, we just stop the debug session. const tokenSource = new CancellationTokenSource(); diff --git a/extensions/ql-vscode/src/query-testing/test-runner.ts b/extensions/ql-vscode/src/query-testing/test-runner.ts index 55b5237b9fb..c9ce940e570 100644 --- a/extensions/ql-vscode/src/query-testing/test-runner.ts +++ b/extensions/ql-vscode/src/query-testing/test-runner.ts @@ -103,6 +103,7 @@ export class TestRunner extends DisposableObject { try { const reopenedDatabase = await this.databaseManager.openDatabase( uri, + closedDatabase.source, false, ); await this.databaseManager.renameDatabaseItem( diff --git a/extensions/ql-vscode/test/factories/databases/databases.ts b/extensions/ql-vscode/test/factories/databases/databases.ts index c874c70536f..c44b471a28b 100644 --- a/extensions/ql-vscode/test/factories/databases/databases.ts +++ b/extensions/ql-vscode/test/factories/databases/databases.ts @@ -11,6 +11,9 @@ export function mockDbOptions(): FullDatabaseOptions { return { dateAdded: 123, language: "", + source: { + type: "folder", + }, }; } diff --git a/extensions/ql-vscode/test/factories/db-config-factories.ts b/extensions/ql-vscode/test/factories/db-config-factories.ts index 11fbd7b9cfd..2708ee24806 100644 --- a/extensions/ql-vscode/test/factories/db-config-factories.ts +++ b/extensions/ql-vscode/test/factories/db-config-factories.ts @@ -7,6 +7,7 @@ import { SelectedDbItem, DB_CONFIG_VERSION, } from "../../src/databases/config/db-config"; +import { DatabaseSource } from "../../src/databases/local-databases/database-source"; export function createDbConfig({ remoteLists = [], @@ -45,16 +46,21 @@ export function createLocalDbConfigItem({ dateAdded = faker.date.past().getTime(), language = `language${faker.number.int()}`, storagePath = `storagePath${faker.number.int()}`, + source = { + type: "folder", + }, }: { name?: string; dateAdded?: number; language?: string; storagePath?: string; + source?: DatabaseSource; } = {}): LocalDatabase { return { name, dateAdded, language, storagePath, + source, }; } diff --git a/extensions/ql-vscode/test/factories/db-item-factories.ts b/extensions/ql-vscode/test/factories/db-item-factories.ts index ff371d0313c..8558ac3afd1 100644 --- a/extensions/ql-vscode/test/factories/db-item-factories.ts +++ b/extensions/ql-vscode/test/factories/db-item-factories.ts @@ -12,6 +12,7 @@ import { RootLocalDbItem, RootRemoteDbItem, } from "../../src/databases/db-item"; +import { DatabaseSource } from "../../src/databases/local-databases/database-source"; // Root Remote Db Items export function createRootRemoteDbItem({ @@ -124,12 +125,16 @@ export function createLocalDatabaseDbItem({ language = `language${faker.number.int()}`, storagePath = `storagePath${faker.number.int()}`, selected = false, + source = { + type: "folder", + }, }: { databaseName?: string; dateAdded?: number; language?: string; storagePath?: string; selected?: boolean; + source?: DatabaseSource; } = {}): LocalDatabaseDbItem { return { kind: DbItemKind.LocalDatabase, @@ -138,6 +143,7 @@ export function createLocalDatabaseDbItem({ dateAdded, language, storagePath, + source, }; } diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts index 296eb8d1e54..4f4e8f97f7c 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts @@ -57,6 +57,9 @@ describe("db item selection", () => { dateAdded: 1234, language: "javascript", storagePath: "/foo/bar", + source: { + type: "folder", + }, selected: true, }); }); diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts index 3b21d095143..7aa70304b50 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts @@ -337,12 +337,18 @@ describe("db tree creator", () => { dateAdded: 1668428293677, language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", + source: { + type: "folder", + }, }, { name: "db2", dateAdded: 1668428472731, language: "cpp", storagePath: "/path/to/db2/", + source: { + type: "folder", + }, }, ], }, @@ -354,6 +360,9 @@ describe("db tree creator", () => { dateAdded: 1668428472731, language: "ruby", storagePath: "/path/to/db3/", + source: { + type: "folder", + }, }, ], }, @@ -380,6 +389,7 @@ describe("db tree creator", () => { databaseName: db.name, dateAdded: db.dateAdded, language: db.language, + source: db.source, storagePath: db.storagePath, parentListName: dbConfig.databases.local.lists[0].name, })), @@ -395,6 +405,7 @@ describe("db tree creator", () => { databaseName: db.name, dateAdded: db.dateAdded, language: db.language, + source: db.source, storagePath: db.storagePath, parentListName: dbConfig.databases.local.lists[1].name, })), @@ -409,12 +420,18 @@ describe("db tree creator", () => { dateAdded: 1668428293677, language: "csharp", storagePath: "/path/to/db1/", + source: { + type: "folder", + }, }, { name: "db2", dateAdded: 1668428472731, language: "go", storagePath: "/path/to/db2/", + source: { + type: "folder", + }, }, ], }); @@ -434,6 +451,7 @@ describe("db tree creator", () => { databaseName: dbConfig.databases.local.databases[0].name, dateAdded: dbConfig.databases.local.databases[0].dateAdded, language: dbConfig.databases.local.databases[0].language, + source: dbConfig.databases.local.databases[0].source, storagePath: dbConfig.databases.local.databases[0].storagePath, }); expect(localDatabaseNodes[1]).toEqual({ @@ -442,6 +460,7 @@ describe("db tree creator", () => { databaseName: dbConfig.databases.local.databases[1].name, dateAdded: dbConfig.databases.local.databases[1].dateAdded, language: dbConfig.databases.local.databases[1].language, + source: dbConfig.databases.local.databases[1].source, storagePath: dbConfig.databases.local.databases[1].storagePath, }); }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts index 8f8c8689fc4..082aeb0b207 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts @@ -186,12 +186,18 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428293677, language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", + source: { + type: "folder", + }, }, { name: "db2", dateAdded: 1668428472731, language: QueryLanguage.Cpp, storagePath: "/path/to/db2/", + source: { + type: "folder", + }, }, ], }, @@ -203,6 +209,9 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428472731, language: "ruby", storagePath: "/path/to/db3/", + source: { + type: "folder", + }, }, ], }, @@ -238,6 +247,9 @@ describe("db panel rendering nodes", () => { language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", selected: false, + source: { + type: "folder", + }, }, { kind: DbItemKind.LocalDatabase, @@ -246,6 +258,9 @@ describe("db panel rendering nodes", () => { language: QueryLanguage.Cpp, storagePath: "/path/to/db2/", selected: false, + source: { + type: "folder", + }, }, ]); checkLocalListItem(localListItems[1], "my-list-2", [ @@ -256,6 +271,9 @@ describe("db panel rendering nodes", () => { language: "ruby", storagePath: "/path/to/db3/", selected: false, + source: { + type: "folder", + }, }, ]); }); @@ -268,12 +286,18 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428293677, language: "csharp", storagePath: "/path/to/db1/", + source: { + type: "folder", + }, }, { name: "db2", dateAdded: 1668428472731, language: "go", storagePath: "/path/to/db2/", + source: { + type: "folder", + }, }, ], }); @@ -306,6 +330,9 @@ describe("db panel rendering nodes", () => { language: "csharp", storagePath: "/path/to/db1/", selected: false, + source: { + type: "folder", + }, }); checkLocalDatabaseItem(localDatabaseItems[1], { kind: DbItemKind.LocalDatabase, @@ -314,6 +341,9 @@ describe("db panel rendering nodes", () => { language: "go", storagePath: "/path/to/db2/", selected: false, + source: { + type: "folder", + }, }); }); }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts index 9b824464741..3058db689fd 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts @@ -597,6 +597,9 @@ describe("local databases", () => { const options: FullDatabaseOptions = { dateAdded: 123, language, + source: { + type: "folder", + }, }; mockDbItem = createMockDB(dir, options); @@ -718,19 +721,28 @@ describe("local databases", () => { }); it("should resolve the database contents", async () => { - await databaseManager.openDatabase(mockDbItem.databaseUri); + await databaseManager.openDatabase( + mockDbItem.databaseUri, + mockDbItem.source, + ); expect(resolveDatabaseContentsSpy).toBeCalledTimes(2); }); it("should set the database as the currently selected one", async () => { - await databaseManager.openDatabase(mockDbItem.databaseUri); + await databaseManager.openDatabase( + mockDbItem.databaseUri, + mockDbItem.source, + ); expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1); }); it("should add database source archive folder", async () => { - await databaseManager.openDatabase(mockDbItem.databaseUri); + await databaseManager.openDatabase( + mockDbItem.databaseUri, + mockDbItem.source, + ); expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1); }); @@ -746,6 +758,7 @@ describe("local databases", () => { await databaseManager.openDatabase( mockDbItem.databaseUri, + mockDbItem.source, makeSelected, nameOverride, { isTutorialDatabase }, @@ -759,7 +772,10 @@ describe("local databases", () => { it("should create a skeleton QL pack", async () => { jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(true); - await databaseManager.openDatabase(mockDbItem.databaseUri); + await databaseManager.openDatabase( + mockDbItem.databaseUri, + mockDbItem.source, + ); expect(createSkeletonPacksSpy).toBeCalledTimes(1); }); @@ -770,7 +786,10 @@ describe("local databases", () => { it("should not create a skeleton QL pack", async () => { jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(false); - await databaseManager.openDatabase(mockDbItem.databaseUri); + await databaseManager.openDatabase( + mockDbItem.databaseUri, + mockDbItem.source, + ); expect(createSkeletonPacksSpy).toBeCalledTimes(0); }); }); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts index ef18b2b011d..11826f544f7 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts @@ -39,12 +39,18 @@ describe("test-runner", () => { const preTestDatabaseItem = new DatabaseItemImpl( Uri.file("/path/to/test/dir/dir.testproj"), undefined, - mockedObject({ displayName: "custom display name" }), + mockedObject({ + displayName: "custom display name", + source: { type: "folder" }, + }), ); const postTestDatabaseItem = new DatabaseItemImpl( Uri.file("/path/to/test/dir/dir.testproj"), undefined, - mockedObject({ displayName: "default name" }), + mockedObject({ + displayName: "default name", + source: { type: "folder" }, + }), ); beforeEach(() => { @@ -160,6 +166,7 @@ describe("test-runner", () => { expect(openDatabaseSpy).toBeCalledTimes(1); expect(openDatabaseSpy).toBeCalledWith( preTestDatabaseItem.databaseUri, + preTestDatabaseItem.source, false, ); From 3ac8a816ef4a4f41ef1b156ce3bf00f4660bbea5 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 13 Nov 2023 14:55:20 +0000 Subject: [PATCH 02/11] Update provenance when inputs change (#3064) --- .../src/model-editor/modeled-method.ts | 35 +++++++++++++++++++ .../view/model-editor/ModelInputDropdown.tsx | 2 ++ .../view/model-editor/ModelKindDropdown.tsx | 2 ++ .../view/model-editor/ModelOutputDropdown.tsx | 2 ++ .../view/model-editor/ModelTypeDropdown.tsx | 14 ++------ .../model-editor/__tests__/MethodRow.spec.tsx | 28 ++++++++++++++- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/extensions/ql-vscode/src/model-editor/modeled-method.ts b/extensions/ql-vscode/src/model-editor/modeled-method.ts index 0d86e8f9f3a..c0a7ab94f48 100644 --- a/extensions/ql-vscode/src/model-editor/modeled-method.ts +++ b/extensions/ql-vscode/src/model-editor/modeled-method.ts @@ -126,3 +126,38 @@ export function isModelAccepted( modeledMethod.provenance !== "ai-generated" ); } + +/** + * Calculates the new provenance for a modeled method based on the current provenance. + * @param modeledMethod The modeled method if there is one. + * @returns The new provenance. + */ +export function calculateNewProvenance( + modeledMethod: ModeledMethod | undefined, +) { + if (!modeledMethod || !modeledMethodSupportsProvenance(modeledMethod)) { + // If nothing has been modeled or the modeled method does not support + // provenance, we assume that the user has entered it manually. + return "manual"; + } + + switch (modeledMethod.provenance) { + case "df-generated": + // If the method has been generated and there has been a change, we assume + // that the user has manually edited it. + return "df-manual"; + case "df-manual": + // If the method has had manual edits, we want the provenance to stay the same. + return "df-manual"; + case "ai-generated": + // If the method has been generated and there has been a change, we assume + // that the user has manually edited it. + return "ai-manual"; + case "ai-manual": + // If the method has had manual edits, we want the provenance to stay the same. + return "ai-manual"; + default: + // The method has been modeled manually. + return "manual"; + } +} diff --git a/extensions/ql-vscode/src/view/model-editor/ModelInputDropdown.tsx b/extensions/ql-vscode/src/view/model-editor/ModelInputDropdown.tsx index ad70a77708c..300d1b296ba 100644 --- a/extensions/ql-vscode/src/view/model-editor/ModelInputDropdown.tsx +++ b/extensions/ql-vscode/src/view/model-editor/ModelInputDropdown.tsx @@ -2,6 +2,7 @@ import * as React from "react"; import { ChangeEvent, useCallback, useMemo } from "react"; import { ModeledMethod, + calculateNewProvenance, isModelAccepted, modeledMethodSupportsInput, } from "../../model-editor/modeled-method"; @@ -53,6 +54,7 @@ export const ModelInputDropdown = ({ onChange({ ...modeledMethod, + provenance: calculateNewProvenance(modeledMethod), input: target.value, }); }, diff --git a/extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx b/extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx index 488e2bb9a12..f1a8c66af49 100644 --- a/extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx +++ b/extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx @@ -5,6 +5,7 @@ import { ModeledMethodKind, modeledMethodSupportsKind, isModelAccepted, + calculateNewProvenance, } from "../../model-editor/modeled-method"; import { getModelsAsDataLanguage } from "../../model-editor/languages"; import { QueryLanguage } from "../../common/query-language"; @@ -52,6 +53,7 @@ export const ModelKindDropdown = ({ onChange({ ...modeledMethod, + provenance: calculateNewProvenance(modeledMethod), kind, }); }, diff --git a/extensions/ql-vscode/src/view/model-editor/ModelOutputDropdown.tsx b/extensions/ql-vscode/src/view/model-editor/ModelOutputDropdown.tsx index 71dac6564ee..970274f992b 100644 --- a/extensions/ql-vscode/src/view/model-editor/ModelOutputDropdown.tsx +++ b/extensions/ql-vscode/src/view/model-editor/ModelOutputDropdown.tsx @@ -2,6 +2,7 @@ import * as React from "react"; import { ChangeEvent, useCallback, useMemo } from "react"; import { ModeledMethod, + calculateNewProvenance, isModelAccepted, modeledMethodSupportsOutput, } from "../../model-editor/modeled-method"; @@ -54,6 +55,7 @@ export const ModelOutputDropdown = ({ onChange({ ...modeledMethod, + provenance: calculateNewProvenance(modeledMethod), output: target.value, }); }, diff --git a/extensions/ql-vscode/src/view/model-editor/ModelTypeDropdown.tsx b/extensions/ql-vscode/src/view/model-editor/ModelTypeDropdown.tsx index e8f36beb815..d6c232d6788 100644 --- a/extensions/ql-vscode/src/view/model-editor/ModelTypeDropdown.tsx +++ b/extensions/ql-vscode/src/view/model-editor/ModelTypeDropdown.tsx @@ -1,11 +1,10 @@ import * as React from "react"; import { ChangeEvent, useCallback } from "react"; import { + calculateNewProvenance, isModelAccepted, ModeledMethod, - modeledMethodSupportsProvenance, ModeledMethodType, - Provenance, } from "../../model-editor/modeled-method"; import { Method } from "../../model-editor/method"; import { createEmptyModeledMethod } from "../../model-editor/modeled-method-empty"; @@ -43,15 +42,6 @@ export const ModelTypeDropdown = ({ (e: ChangeEvent) => { const modelsAsDataLanguage = getModelsAsDataLanguage(language); - let newProvenance: Provenance = "manual"; - if (modeledMethod && modeledMethodSupportsProvenance(modeledMethod)) { - if (modeledMethod.provenance === "df-generated") { - newProvenance = "df-manual"; - } else if (modeledMethod.provenance === "ai-generated") { - newProvenance = "ai-manual"; - } - } - const emptyModeledMethod = createEmptyModeledMethod( e.target.value as ModeledMethodType, method, @@ -67,7 +57,7 @@ export const ModelTypeDropdown = ({ updatedModeledMethod.output = "ReturnValue"; } if ("provenance" in updatedModeledMethod) { - updatedModeledMethod.provenance = newProvenance; + updatedModeledMethod.provenance = calculateNewProvenance(modeledMethod); } if ("kind" in updatedModeledMethod) { updatedModeledMethod.kind = "value"; diff --git a/extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx b/extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx index bdd303e9838..2d15d213647 100644 --- a/extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx +++ b/extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx @@ -27,7 +27,7 @@ describe(MethodRow.name, () => { input: "Argument[0]", output: "ReturnValue", kind: "taint", - provenance: "df-generated", + provenance: "manual", }; const onChange = jest.fn(); @@ -111,6 +111,32 @@ describe(MethodRow.name, () => { ]); }); + it("changes the provenance when the kind is changed", async () => { + const modeledMethodWithGeneratedProvenance: ModeledMethod = { + ...modeledMethod, + provenance: "df-generated", + }; + render({ modeledMethods: [modeledMethodWithGeneratedProvenance] }); + + onChange.mockReset(); + + expect(screen.getByRole("combobox", { name: "Kind" })).toHaveValue("taint"); + + await userEvent.selectOptions( + screen.getByRole("combobox", { name: "Kind" }), + "value", + ); + + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange).toHaveBeenCalledWith(method.signature, [ + { + ...modeledMethod, + kind: "value", + provenance: "df-manual", + }, + ]); + }); + it("has the correct input options", () => { render(); From 5770eda46630e7a644efb64f70fa47c86d9c64c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:16:43 +0000 Subject: [PATCH 03/11] Bump CLI version from v2.15.1 to v2.15.2 for integration tests (#3067) --- extensions/ql-vscode/supported_cli_versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/supported_cli_versions.json b/extensions/ql-vscode/supported_cli_versions.json index 290e5e29964..5d3a6aac4bc 100644 --- a/extensions/ql-vscode/supported_cli_versions.json +++ b/extensions/ql-vscode/supported_cli_versions.json @@ -1,5 +1,5 @@ [ - "v2.15.1", + "v2.15.2", "v2.14.6", "v2.13.5", "v2.12.7", From 9e914c9ba17e4d9d61b10b74d53426f3e8392a0a Mon Sep 17 00:00:00 2001 From: Shati Patel <42641846+shati-patel@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:04:53 +0000 Subject: [PATCH 04/11] Don't add database source archive folders by default (#3047) --- extensions/ql-vscode/CHANGELOG.md | 4 ++++ extensions/ql-vscode/package.json | 14 ++++++++++++-- extensions/ql-vscode/src/config.ts | 13 +++++++++++-- .../ql-vscode/src/databases/database-fetcher.ts | 10 +++++----- .../databases/local-databases/database-manager.ts | 5 +++-- .../local-queries/local-databases.test.ts | 12 +++++++++++- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index 932c04fd2ba..6018434b834 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,6 +2,10 @@ ## [UNRELEASED] +- When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047) + - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view. + - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`. +- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) - The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055) ## 1.9.4 - 6 November 2023 diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index ea0a2d28c9f..a440f7139a3 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -372,13 +372,23 @@ }, { "type": "object", - "title": "Downloading databases", + "title": "Adding databases", "order": 6, "properties": { + "codeQL.addingDatabases.allowHttp": { + "type": "boolean", + "default": false, + "description": "Allow databases to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers." + }, "codeQL.databaseDownload.allowHttp": { + "type": "boolean", + "markdownDeprecationMessage": "**Deprecated**: Please use `#codeQL.addingDatabases.allowHttp#` instead.", + "deprecationMessage": "Deprecated: Please use codeQL.addingDatabases.allowHttp instead." + }, + "codeQL.addingDatabases.addDatabaseSourceToWorkspace": { "type": "boolean", "default": false, - "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers." + "markdownDescription": "When adding a CodeQL database, automatically add the database's source folder as a workspace folder. Warning: enabling this option in a single-folder workspace will cause the workspace to reload as a [multi-root workspace](https://code.visualstudio.com/docs/editor/multi-root-workspaces). This may cause query history and database lists to be reset." } } }, diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts index b2aac995d54..a13e1f50ad8 100644 --- a/extensions/ql-vscode/src/config.ts +++ b/extensions/ql-vscode/src/config.ts @@ -641,14 +641,23 @@ export function isCodespacesTemplate() { return !!CODESPACES_TEMPLATE.getValue(); } -const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING); +const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING); -const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING); +const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING); export function allowHttp(): boolean { return ALLOW_HTTP_SETTING.getValue() || false; } +const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting( + "addDatabaseSourceToWorkspace", + ADDING_DATABASES_SETTING, +); + +export function addDatabaseSourceToWorkspace(): boolean { + return ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING.getValue() || false; +} + /** * Parent setting for all settings related to the "Create Query" command. */ diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index 5521dcd00c3..279b9a59f18 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -29,7 +29,7 @@ import { } from "../common/github-url-identifier-helper"; import { Credentials } from "../common/authentication"; import { AppCommandManager } from "../common/commands"; -import { allowHttp } from "../config"; +import { addDatabaseSourceToWorkspace, allowHttp } from "../config"; import { showAndLogInformationMessage } from "../common/logging"; import { AppOctokit } from "../common/octokit"; import { getLanguageDisplayName } from "../common/query-language"; @@ -99,7 +99,7 @@ export async function promptImportGithubDatabase( cli?: CodeQLCliServer, language?: string, makeSelected = true, - addSourceArchiveFolder = true, + addSourceArchiveFolder = addDatabaseSourceToWorkspace(), ): Promise { const githubRepo = await askForGitHubRepo(progress); if (!githubRepo) { @@ -178,7 +178,7 @@ export async function downloadGitHubDatabase( cli?: CodeQLCliServer, language?: string, makeSelected = true, - addSourceArchiveFolder = true, + addSourceArchiveFolder = addDatabaseSourceToWorkspace(), ): Promise { const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo; if (!isValidGitHubNwo(nwo)) { @@ -295,7 +295,7 @@ async function databaseArchiveFetcher( progress: ProgressCallback, cli?: CodeQLCliServer, makeSelected = true, - addSourceArchiveFolder = true, + addSourceArchiveFolder = addDatabaseSourceToWorkspace(), ): Promise { progress({ message: "Getting database", @@ -476,7 +476,7 @@ async function checkForFailingResponse( return response; } - // An error downloading the database. Attempt to extract the resaon behind it. + // An error downloading the database. Attempt to extract the reason behind it. const text = await response.text(); let msg: string; try { diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index 1e58a5e2858..c4d3cde1146 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -7,6 +7,7 @@ import { QueryRunner } from "../../query-server"; import * as cli from "../../codeql-cli/cli"; import { ProgressCallback, withProgress } from "../../common/vscode/progress"; import { + addDatabaseSourceToWorkspace, getAutogenerateQlPacks, isCodespacesTemplate, setAutogenerateQlPacks, @@ -135,7 +136,7 @@ export class DatabaseManager extends DisposableObject { displayName?: string, { isTutorialDatabase = false, - addSourceArchiveFolder = true, + addSourceArchiveFolder = addDatabaseSourceToWorkspace(), }: OpenDatabaseOptions = {}, ): Promise { const databaseItem = await this.createDatabaseItem(uri, displayName); @@ -158,7 +159,7 @@ export class DatabaseManager extends DisposableObject { databaseItem: DatabaseItemImpl, makeSelected: boolean, isTutorialDatabase?: boolean, - addSourceArchiveFolder = true, + addSourceArchiveFolder = addDatabaseSourceToWorkspace(), ): Promise { const existingItem = this.findDatabaseItem(databaseItem.databaseUri); if (existingItem !== undefined) { diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts index f494238c27a..445dbb0dac1 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts @@ -739,7 +739,17 @@ describe("local databases", () => { expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1); }); - it("should add database source archive folder", async () => { + it("should not add database source archive folder when `codeQL.addingDatabases.addDatabaseSourceToWorkspace` is `false`", async () => { + jest.spyOn(config, "addDatabaseSourceToWorkspace").mockReturnValue(false); + + await databaseManager.openDatabase(mockDbItem.databaseUri); + + expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(0); + }); + + it("should add database source archive folder when `codeQL.addingDatabases.addDatabaseSourceToWorkspace` is `true`", async () => { + jest.spyOn(config, "addDatabaseSourceToWorkspace").mockReturnValue(true); + await databaseManager.openDatabase(mockDbItem.databaseUri); expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1); From 5df9dfc78ab27d1425e30cc9004de461ba9f72c0 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 14 Nov 2023 10:04:05 +0100 Subject: [PATCH 05/11] Rename source to origin --- .../src/databases/config/db-config.ts | 4 +-- .../src/databases/database-fetcher.ts | 8 ++--- extensions/ql-vscode/src/databases/db-item.ts | 4 +-- .../src/databases/db-tree-creator.ts | 2 +- .../local-databases/database-item-impl.ts | 6 ++-- .../local-databases/database-item.ts | 6 ++-- .../local-databases/database-manager.ts | 20 ++++++------- .../local-databases/database-options.ts | 6 ++-- .../local-databases/database-origin.ts | 30 +++++++++++++++++++ .../local-databases/database-source.ts | 30 ------------------- .../src/query-testing/test-runner.ts | 2 +- .../test/factories/databases/databases.ts | 2 +- .../test/factories/db-config-factories.ts | 8 ++--- .../test/factories/db-item-factories.ts | 8 ++--- .../databases/db-item-selection.test.ts | 2 +- .../databases/db-tree-creator.test.ts | 18 +++++------ .../databases/db-panel-rendering.test.ts | 20 ++++++------- .../local-queries/local-databases.test.ts | 14 ++++----- .../query-testing/test-runner.test.ts | 6 ++-- 19 files changed, 98 insertions(+), 98 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-origin.ts delete mode 100644 extensions/ql-vscode/src/databases/local-databases/database-source.ts diff --git a/extensions/ql-vscode/src/databases/config/db-config.ts b/extensions/ql-vscode/src/databases/config/db-config.ts index 7825f17e92c..792d63e547c 100644 --- a/extensions/ql-vscode/src/databases/config/db-config.ts +++ b/extensions/ql-vscode/src/databases/config/db-config.ts @@ -1,7 +1,7 @@ // Contains models and consts for the data we want to store in the database config. // Changes to these models should be done carefully and account for backwards compatibility of data. -import { DatabaseSource } from "../local-databases/database-source"; +import { DatabaseOrigin } from "../local-databases/database-origin"; export const DB_CONFIG_VERSION = 1; @@ -90,7 +90,7 @@ export interface LocalDatabase { name: string; dateAdded: number; language: string; - source: DatabaseSource; + origin: DatabaseOrigin; storagePath: string; } diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index 3e2f5106738..df1a7aebfd0 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -33,7 +33,7 @@ import { allowHttp } from "../config"; import { showAndLogInformationMessage } from "../common/logging"; import { AppOctokit } from "../common/octokit"; import { getLanguageDisplayName } from "../common/query-language"; -import { DatabaseSource } from "./local-databases/database-source"; +import { DatabaseOrigin } from "./local-databases/database-origin"; /** * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file. @@ -296,7 +296,7 @@ export async function importArchiveDatabase( * @param databaseManager the DatabaseManager * @param storagePath where to store the unzipped database. * @param nameOverride a name for the database that overrides the default - * @param source the source of the database + * @param origin the origin of the database * @param progress callback to send progress messages to * @param makeSelected make the new database selected in the databases panel (default: true) * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace @@ -307,7 +307,7 @@ async function databaseArchiveFetcher( databaseManager: DatabaseManager, storagePath: string, nameOverride: string | undefined, - source: DatabaseSource, + origin: DatabaseOrigin, progress: ProgressCallback, cli?: CodeQLCliServer, makeSelected = true, @@ -352,7 +352,7 @@ async function databaseArchiveFetcher( const item = await databaseManager.openDatabase( Uri.file(dbPath), - source, + origin, makeSelected, nameOverride, { diff --git a/extensions/ql-vscode/src/databases/db-item.ts b/extensions/ql-vscode/src/databases/db-item.ts index d2c98717526..a19eed9b827 100644 --- a/extensions/ql-vscode/src/databases/db-item.ts +++ b/extensions/ql-vscode/src/databases/db-item.ts @@ -1,6 +1,6 @@ // This file contains models that are used to represent the databases. -import { DatabaseSource } from "./local-databases/database-source"; +import { DatabaseOrigin } from "./local-databases/database-origin"; export enum DbItemKind { RootLocal = "RootLocal", @@ -40,7 +40,7 @@ export interface LocalDatabaseDbItem { databaseName: string; dateAdded: number; language: string; - source: DatabaseSource; + origin: DatabaseOrigin; storagePath: string; parentListName?: string; } diff --git a/extensions/ql-vscode/src/databases/db-tree-creator.ts b/extensions/ql-vscode/src/databases/db-tree-creator.ts index a804603c895..01619558a8b 100644 --- a/extensions/ql-vscode/src/databases/db-tree-creator.ts +++ b/extensions/ql-vscode/src/databases/db-tree-creator.ts @@ -197,7 +197,7 @@ function createLocalDb( databaseName: db.name, dateAdded: db.dateAdded, language: db.language, - source: db.source, + origin: db.origin, storagePath: db.storagePath, selected: !!selected, parentListName: listName, diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts index 76d98259423..0182213da10 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -14,7 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics"; import { stat } from "fs-extra"; import { containsPath, pathsEqual } from "../../common/files"; import { DatabaseContents } from "./database-contents"; -import { DatabaseSource } from "./database-source"; +import { DatabaseOrigin } from "./database-origin"; export class DatabaseItemImpl implements DatabaseItem { // These are only public in the implementation, they are readonly in the interface @@ -62,8 +62,8 @@ export class DatabaseItemImpl implements DatabaseItem { return this.options.dateAdded; } - public get source(): DatabaseSource | undefined { - return this.options.source; + public get origin(): DatabaseOrigin | undefined { + return this.options.origin; } public resolveSourceFile(uriStr: string | undefined): vscode.Uri { diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts index e9774a4d4f0..ebc1499257c 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts @@ -2,7 +2,7 @@ import vscode from "vscode"; import * as cli from "../../codeql-cli/cli"; import { DatabaseContents } from "./database-contents"; import { DatabaseOptions } from "./database-options"; -import { DatabaseSource } from "./database-source"; +import { DatabaseOrigin } from "./database-origin"; /** An item in the list of available databases */ export interface DatabaseItem { @@ -27,9 +27,9 @@ export interface DatabaseItem { readonly dateAdded: number | undefined; /** - * The source this database item was added from or undefined if unknown. + * The origin this database item was retrieved from or undefined if unknown. */ - readonly source: DatabaseSource | undefined; + readonly origin: DatabaseOrigin | undefined; /** If the database is invalid, describes why. */ readonly error: Error | undefined; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index 7cfc5ef409e..0ead2eada58 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -34,7 +34,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; import { DatabaseResolver } from "./database-resolver"; import { telemetryListener } from "../../common/vscode/telemetry"; import { LanguageContextStore } from "../../language-context-store"; -import { DatabaseSource } from "./database-source"; +import { DatabaseOrigin } from "./database-origin"; /** * The name of the key in the workspaceState dictionary in which we @@ -132,7 +132,7 @@ export class DatabaseManager extends DisposableObject { */ public async openDatabase( uri: vscode.Uri, - source: DatabaseSource | undefined, + origin: DatabaseOrigin | undefined, makeSelected = true, displayName?: string, { @@ -142,7 +142,7 @@ export class DatabaseManager extends DisposableObject { ): Promise { const databaseItem = await this.createDatabaseItem( uri, - source, + origin, displayName, ); @@ -195,7 +195,7 @@ export class DatabaseManager extends DisposableObject { */ private async createDatabaseItem( uri: vscode.Uri, - source: DatabaseSource | undefined, + origin: DatabaseOrigin | undefined, displayName: string | undefined, ): Promise { const contents = await DatabaseResolver.resolveDatabaseContents(uri); @@ -204,7 +204,7 @@ export class DatabaseManager extends DisposableObject { displayName, dateAdded: Date.now(), language: await this.getPrimaryLanguage(uri.fsPath), - source, + origin, }; const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions); @@ -220,7 +220,7 @@ export class DatabaseManager extends DisposableObject { */ public async createOrOpenDatabaseItem( uri: vscode.Uri, - source: DatabaseSource | undefined, + origin: DatabaseOrigin | undefined, ): Promise { const existingItem = this.findDatabaseItem(uri); if (existingItem !== undefined) { @@ -229,7 +229,7 @@ export class DatabaseManager extends DisposableObject { } // We don't add this to the list automatically, but the user can add it later. - return this.createDatabaseItem(uri, source, undefined); + return this.createDatabaseItem(uri, origin, undefined); } public async createSkeletonPacks(databaseItem: DatabaseItem) { @@ -364,7 +364,7 @@ export class DatabaseManager extends DisposableObject { let displayName: string | undefined = undefined; let dateAdded = undefined; let language = undefined; - let source = undefined; + let origin = undefined; if (state.options) { if (typeof state.options.displayName === "string") { displayName = state.options.displayName; @@ -373,7 +373,7 @@ export class DatabaseManager extends DisposableObject { dateAdded = state.options.dateAdded; } language = state.options.language; - source = state.options.source; + origin = state.options.origin; } const dbBaseUri = vscode.Uri.parse(state.uri, true); @@ -386,7 +386,7 @@ export class DatabaseManager extends DisposableObject { displayName, dateAdded, language, - source, + origin, }; const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions); diff --git a/extensions/ql-vscode/src/databases/local-databases/database-options.ts b/extensions/ql-vscode/src/databases/local-databases/database-options.ts index 2dc97e1e4e0..ffbc30378d8 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-options.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-options.ts @@ -1,14 +1,14 @@ -import { DatabaseSource } from "./database-source"; +import { DatabaseOrigin } from "./database-origin"; export interface DatabaseOptions { displayName?: string; dateAdded?: number | undefined; language?: string; - source?: DatabaseSource; + origin?: DatabaseOrigin; } export interface FullDatabaseOptions extends DatabaseOptions { dateAdded: number | undefined; language: string | undefined; - source: DatabaseSource | undefined; + origin: DatabaseOrigin | undefined; } diff --git a/extensions/ql-vscode/src/databases/local-databases/database-origin.ts b/extensions/ql-vscode/src/databases/local-databases/database-origin.ts new file mode 100644 index 00000000000..2b397f4277d --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-origin.ts @@ -0,0 +1,30 @@ +interface DatabaseOriginFolder { + type: "folder"; +} + +interface DatabaseOriginArchive { + type: "archive"; + path: string; +} + +interface DatabaseOriginGitHub { + type: "github"; + repository: string; + commitOid: string | null; +} + +interface DatabaseOriginInternet { + type: "url"; + url: string; +} + +interface DatabaseOriginDebugger { + type: "debugger"; +} + +export type DatabaseOrigin = + | DatabaseOriginFolder + | DatabaseOriginArchive + | DatabaseOriginGitHub + | DatabaseOriginInternet + | DatabaseOriginDebugger; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-source.ts b/extensions/ql-vscode/src/databases/local-databases/database-source.ts deleted file mode 100644 index 9afbafafabf..00000000000 --- a/extensions/ql-vscode/src/databases/local-databases/database-source.ts +++ /dev/null @@ -1,30 +0,0 @@ -interface DatabaseSourceFolder { - type: "folder"; -} - -interface DatabaseSourceArchive { - type: "archive"; - path: string; -} - -interface DatabaseSourceGitHub { - type: "github"; - repository: string; - commitOid: string | null; -} - -interface DatabaseSourceInternet { - type: "url"; - url: string; -} - -interface DatabaseSourceDebugger { - type: "debugger"; -} - -export type DatabaseSource = - | DatabaseSourceFolder - | DatabaseSourceArchive - | DatabaseSourceGitHub - | DatabaseSourceInternet - | DatabaseSourceDebugger; diff --git a/extensions/ql-vscode/src/query-testing/test-runner.ts b/extensions/ql-vscode/src/query-testing/test-runner.ts index c9ce940e570..b519d2ad228 100644 --- a/extensions/ql-vscode/src/query-testing/test-runner.ts +++ b/extensions/ql-vscode/src/query-testing/test-runner.ts @@ -103,7 +103,7 @@ export class TestRunner extends DisposableObject { try { const reopenedDatabase = await this.databaseManager.openDatabase( uri, - closedDatabase.source, + closedDatabase.origin, false, ); await this.databaseManager.renameDatabaseItem( diff --git a/extensions/ql-vscode/test/factories/databases/databases.ts b/extensions/ql-vscode/test/factories/databases/databases.ts index c44b471a28b..006c5d480de 100644 --- a/extensions/ql-vscode/test/factories/databases/databases.ts +++ b/extensions/ql-vscode/test/factories/databases/databases.ts @@ -11,7 +11,7 @@ export function mockDbOptions(): FullDatabaseOptions { return { dateAdded: 123, language: "", - source: { + origin: { type: "folder", }, }; diff --git a/extensions/ql-vscode/test/factories/db-config-factories.ts b/extensions/ql-vscode/test/factories/db-config-factories.ts index 2708ee24806..2bc6a5046f9 100644 --- a/extensions/ql-vscode/test/factories/db-config-factories.ts +++ b/extensions/ql-vscode/test/factories/db-config-factories.ts @@ -7,7 +7,7 @@ import { SelectedDbItem, DB_CONFIG_VERSION, } from "../../src/databases/config/db-config"; -import { DatabaseSource } from "../../src/databases/local-databases/database-source"; +import { DatabaseOrigin } from "../../src/databases/local-databases/database-origin"; export function createDbConfig({ remoteLists = [], @@ -46,7 +46,7 @@ export function createLocalDbConfigItem({ dateAdded = faker.date.past().getTime(), language = `language${faker.number.int()}`, storagePath = `storagePath${faker.number.int()}`, - source = { + origin = { type: "folder", }, }: { @@ -54,13 +54,13 @@ export function createLocalDbConfigItem({ dateAdded?: number; language?: string; storagePath?: string; - source?: DatabaseSource; + origin?: DatabaseOrigin; } = {}): LocalDatabase { return { name, dateAdded, language, storagePath, - source, + origin, }; } diff --git a/extensions/ql-vscode/test/factories/db-item-factories.ts b/extensions/ql-vscode/test/factories/db-item-factories.ts index 8558ac3afd1..3c01c1d4178 100644 --- a/extensions/ql-vscode/test/factories/db-item-factories.ts +++ b/extensions/ql-vscode/test/factories/db-item-factories.ts @@ -12,7 +12,7 @@ import { RootLocalDbItem, RootRemoteDbItem, } from "../../src/databases/db-item"; -import { DatabaseSource } from "../../src/databases/local-databases/database-source"; +import { DatabaseOrigin } from "../../src/databases/local-databases/database-origin"; // Root Remote Db Items export function createRootRemoteDbItem({ @@ -125,7 +125,7 @@ export function createLocalDatabaseDbItem({ language = `language${faker.number.int()}`, storagePath = `storagePath${faker.number.int()}`, selected = false, - source = { + origin = { type: "folder", }, }: { @@ -134,7 +134,7 @@ export function createLocalDatabaseDbItem({ language?: string; storagePath?: string; selected?: boolean; - source?: DatabaseSource; + origin?: DatabaseOrigin; } = {}): LocalDatabaseDbItem { return { kind: DbItemKind.LocalDatabase, @@ -143,7 +143,7 @@ export function createLocalDatabaseDbItem({ dateAdded, language, storagePath, - source, + origin, }; } diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts index 4f4e8f97f7c..e2a6d2631d1 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts @@ -57,7 +57,7 @@ describe("db item selection", () => { dateAdded: 1234, language: "javascript", storagePath: "/foo/bar", - source: { + origin: { type: "folder", }, selected: true, diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts index 7aa70304b50..5170d66e330 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts @@ -337,7 +337,7 @@ describe("db tree creator", () => { dateAdded: 1668428293677, language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", - source: { + origin: { type: "folder", }, }, @@ -346,7 +346,7 @@ describe("db tree creator", () => { dateAdded: 1668428472731, language: "cpp", storagePath: "/path/to/db2/", - source: { + origin: { type: "folder", }, }, @@ -360,7 +360,7 @@ describe("db tree creator", () => { dateAdded: 1668428472731, language: "ruby", storagePath: "/path/to/db3/", - source: { + origin: { type: "folder", }, }, @@ -389,7 +389,7 @@ describe("db tree creator", () => { databaseName: db.name, dateAdded: db.dateAdded, language: db.language, - source: db.source, + origin: db.origin, storagePath: db.storagePath, parentListName: dbConfig.databases.local.lists[0].name, })), @@ -405,7 +405,7 @@ describe("db tree creator", () => { databaseName: db.name, dateAdded: db.dateAdded, language: db.language, - source: db.source, + origin: db.origin, storagePath: db.storagePath, parentListName: dbConfig.databases.local.lists[1].name, })), @@ -420,7 +420,7 @@ describe("db tree creator", () => { dateAdded: 1668428293677, language: "csharp", storagePath: "/path/to/db1/", - source: { + origin: { type: "folder", }, }, @@ -429,7 +429,7 @@ describe("db tree creator", () => { dateAdded: 1668428472731, language: "go", storagePath: "/path/to/db2/", - source: { + origin: { type: "folder", }, }, @@ -451,7 +451,7 @@ describe("db tree creator", () => { databaseName: dbConfig.databases.local.databases[0].name, dateAdded: dbConfig.databases.local.databases[0].dateAdded, language: dbConfig.databases.local.databases[0].language, - source: dbConfig.databases.local.databases[0].source, + origin: dbConfig.databases.local.databases[0].origin, storagePath: dbConfig.databases.local.databases[0].storagePath, }); expect(localDatabaseNodes[1]).toEqual({ @@ -460,7 +460,7 @@ describe("db tree creator", () => { databaseName: dbConfig.databases.local.databases[1].name, dateAdded: dbConfig.databases.local.databases[1].dateAdded, language: dbConfig.databases.local.databases[1].language, - source: dbConfig.databases.local.databases[1].source, + origin: dbConfig.databases.local.databases[1].origin, storagePath: dbConfig.databases.local.databases[1].storagePath, }); }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts index 082aeb0b207..2652c362c43 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts @@ -186,7 +186,7 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428293677, language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", - source: { + origin: { type: "folder", }, }, @@ -195,7 +195,7 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428472731, language: QueryLanguage.Cpp, storagePath: "/path/to/db2/", - source: { + origin: { type: "folder", }, }, @@ -209,7 +209,7 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428472731, language: "ruby", storagePath: "/path/to/db3/", - source: { + origin: { type: "folder", }, }, @@ -247,7 +247,7 @@ describe("db panel rendering nodes", () => { language: QueryLanguage.Cpp, storagePath: "/path/to/db1/", selected: false, - source: { + origin: { type: "folder", }, }, @@ -258,7 +258,7 @@ describe("db panel rendering nodes", () => { language: QueryLanguage.Cpp, storagePath: "/path/to/db2/", selected: false, - source: { + origin: { type: "folder", }, }, @@ -271,7 +271,7 @@ describe("db panel rendering nodes", () => { language: "ruby", storagePath: "/path/to/db3/", selected: false, - source: { + origin: { type: "folder", }, }, @@ -286,7 +286,7 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428293677, language: "csharp", storagePath: "/path/to/db1/", - source: { + origin: { type: "folder", }, }, @@ -295,7 +295,7 @@ describe("db panel rendering nodes", () => { dateAdded: 1668428472731, language: "go", storagePath: "/path/to/db2/", - source: { + origin: { type: "folder", }, }, @@ -330,7 +330,7 @@ describe("db panel rendering nodes", () => { language: "csharp", storagePath: "/path/to/db1/", selected: false, - source: { + origin: { type: "folder", }, }); @@ -341,7 +341,7 @@ describe("db panel rendering nodes", () => { language: "go", storagePath: "/path/to/db2/", selected: false, - source: { + origin: { type: "folder", }, }); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts index 3058db689fd..8a9517b23ca 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts @@ -597,7 +597,7 @@ describe("local databases", () => { const options: FullDatabaseOptions = { dateAdded: 123, language, - source: { + origin: { type: "folder", }, }; @@ -723,7 +723,7 @@ describe("local databases", () => { it("should resolve the database contents", async () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, ); expect(resolveDatabaseContentsSpy).toBeCalledTimes(2); @@ -732,7 +732,7 @@ describe("local databases", () => { it("should set the database as the currently selected one", async () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, ); expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1); @@ -741,7 +741,7 @@ describe("local databases", () => { it("should add database source archive folder", async () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, ); expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1); @@ -758,7 +758,7 @@ describe("local databases", () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, makeSelected, nameOverride, { isTutorialDatabase }, @@ -774,7 +774,7 @@ describe("local databases", () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, ); expect(createSkeletonPacksSpy).toBeCalledTimes(1); @@ -788,7 +788,7 @@ describe("local databases", () => { await databaseManager.openDatabase( mockDbItem.databaseUri, - mockDbItem.source, + mockDbItem.origin, ); expect(createSkeletonPacksSpy).toBeCalledTimes(0); }); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts index 11826f544f7..5de4233a479 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts @@ -41,7 +41,7 @@ describe("test-runner", () => { undefined, mockedObject({ displayName: "custom display name", - source: { type: "folder" }, + origin: { type: "folder" }, }), ); const postTestDatabaseItem = new DatabaseItemImpl( @@ -49,7 +49,7 @@ describe("test-runner", () => { undefined, mockedObject({ displayName: "default name", - source: { type: "folder" }, + origin: { type: "folder" }, }), ); @@ -166,7 +166,7 @@ describe("test-runner", () => { expect(openDatabaseSpy).toBeCalledTimes(1); expect(openDatabaseSpy).toBeCalledWith( preTestDatabaseItem.databaseUri, - preTestDatabaseItem.source, + preTestDatabaseItem.origin, false, ); From be2891dfbaee4795a5d894acfafc4fa59333904e Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Tue, 14 Nov 2023 11:38:28 +0000 Subject: [PATCH 06/11] Replace ModelEditorViewTracker with modeling store and events (#3065) --- .../method-modeling/method-modeling-panel.ts | 3 -- .../method-modeling-view-provider.ts | 6 +-- .../src/model-editor/model-editor-module.ts | 31 +++++--------- .../model-editor/model-editor-view-tracker.ts | 33 --------------- .../src/model-editor/model-editor-view.ts | 40 ++++++++++--------- .../src/model-editor/modeling-events.ts | 36 +++++++++++++++++ .../src/model-editor/modeling-store.ts | 4 ++ .../modelEditorViewTrackerMock.ts | 19 --------- .../model-editor/modelingEventsMock.ts | 6 +++ .../model-editor/model-editor-view.test.ts | 3 -- 10 files changed, 79 insertions(+), 102 deletions(-) delete mode 100644 extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts delete mode 100644 extensions/ql-vscode/test/__mocks__/model-editor/modelEditorViewTrackerMock.ts diff --git a/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts b/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts index 2cadeaa59a9..ec27fbe59ea 100644 --- a/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts +++ b/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts @@ -4,7 +4,6 @@ import { DisposableObject } from "../../common/disposable-object"; import { MethodModelingViewProvider } from "./method-modeling-view-provider"; import { Method } from "../method"; import { ModelingStore } from "../modeling-store"; -import { ModelEditorViewTracker } from "../model-editor-view-tracker"; import { ModelConfigListener } from "../../config"; import { DatabaseItem } from "../../databases/local-databases"; import { ModelingEvents } from "../modeling-events"; @@ -16,7 +15,6 @@ export class MethodModelingPanel extends DisposableObject { app: App, modelingStore: ModelingStore, modelingEvents: ModelingEvents, - editorViewTracker: ModelEditorViewTracker, ) { super(); @@ -29,7 +27,6 @@ export class MethodModelingPanel extends DisposableObject { app, modelingStore, modelingEvents, - editorViewTracker, modelConfig, ); this.push( diff --git a/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts b/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts index eb2bef583de..73fc3a2981f 100644 --- a/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts +++ b/extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts @@ -10,7 +10,6 @@ import { Method } from "../method"; import { ModelingStore } from "../modeling-store"; import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider"; import { assertNever } from "../../common/helpers-pure"; -import { ModelEditorViewTracker } from "../model-editor-view-tracker"; import { ModelConfigListener } from "../../config"; import { DatabaseItem } from "../../databases/local-databases"; import { ModelingEvents } from "../modeling-events"; @@ -33,7 +32,6 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider< app: App, private readonly modelingStore: ModelingStore, private readonly modelingEvents: ModelingEvents, - private readonly editorViewTracker: ModelEditorViewTracker, private readonly modelConfig: ModelConfigListener, ) { super(app, "method-modeling"); @@ -158,10 +156,10 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider< return; } - const view = this.editorViewTracker.getView( + this.modelingEvents.fireRevealInModelEditorEvent( this.databaseItem.databaseUri.toString(), + method, ); - await view?.revealMethod(method); } private registerToModelingEvents(): void { diff --git a/extensions/ql-vscode/src/model-editor/model-editor-module.ts b/extensions/ql-vscode/src/model-editor/model-editor-module.ts index 1208b26297d..4388aeb9345 100644 --- a/extensions/ql-vscode/src/model-editor/model-editor-module.ts +++ b/extensions/ql-vscode/src/model-editor/model-editor-module.ts @@ -19,7 +19,6 @@ import { setUpPack } from "./model-editor-queries-setup"; import { MethodModelingPanel } from "./method-modeling/method-modeling-panel"; import { ModelingStore } from "./modeling-store"; import { showResolvableLocation } from "../databases/local-databases/locations"; -import { ModelEditorViewTracker } from "./model-editor-view-tracker"; import { ModelConfigListener } from "../config"; import { ModelingEvents } from "./modeling-events"; import { getModelsAsDataLanguage } from "./languages"; @@ -30,7 +29,6 @@ export class ModelEditorModule extends DisposableObject { private readonly queryStorageDir: string; private readonly modelingStore: ModelingStore; private readonly modelingEvents: ModelingEvents; - private readonly editorViewTracker: ModelEditorViewTracker; private readonly methodsUsagePanel: MethodsUsagePanel; private readonly methodModelingPanel: MethodModelingPanel; private readonly modelConfig: ModelConfigListener; @@ -46,17 +44,11 @@ export class ModelEditorModule extends DisposableObject { this.queryStorageDir = join(baseQueryStorageDir, "model-editor-results"); this.modelingEvents = new ModelingEvents(app); this.modelingStore = new ModelingStore(this.modelingEvents); - this.editorViewTracker = new ModelEditorViewTracker(); this.methodsUsagePanel = this.push( new MethodsUsagePanel(this.modelingStore, this.modelingEvents, cliServer), ); this.methodModelingPanel = this.push( - new MethodModelingPanel( - app, - this.modelingStore, - this.modelingEvents, - this.editorViewTracker, - ), + new MethodModelingPanel(app, this.modelingStore, this.modelingEvents), ); this.modelConfig = this.push(new ModelConfigListener()); @@ -144,12 +136,10 @@ export class ModelEditorModule extends DisposableObject { const initialMode = definition.availableModes?.[0] ?? INITIAL_MODE; - const existingView = this.editorViewTracker.getView( - db.databaseUri.toString(), - ); - if (existingView) { - await existingView.focusView(); - + if (this.modelingStore.isDbOpen(db.databaseUri.toString())) { + this.modelingEvents.fireFocusModelEditorEvent( + db.databaseUri.toString(), + ); return; } @@ -218,12 +208,10 @@ export class ModelEditorModule extends DisposableObject { // Check again just before opening the editor to ensure no model editor has been opened between // our first check and now. - const existingView = this.editorViewTracker.getView( - db.databaseUri.toString(), - ); - if (existingView) { - await existingView.focusView(); - + if (this.modelingStore.isDbOpen(db.databaseUri.toString())) { + this.modelingEvents.fireFocusModelEditorEvent( + db.databaseUri.toString(), + ); return; } @@ -231,7 +219,6 @@ export class ModelEditorModule extends DisposableObject { this.app, this.modelingStore, this.modelingEvents, - this.editorViewTracker, this.modelConfig, this.databaseManager, this.cliServer, diff --git a/extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts b/extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts deleted file mode 100644 index b4cf12978de..00000000000 --- a/extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Method } from "./method"; - -interface ModelEditorViewInterface { - databaseUri: string; - - revealMethod(method: Method): Promise; -} - -export class ModelEditorViewTracker< - T extends ModelEditorViewInterface = ModelEditorViewInterface, -> { - private readonly views = new Map(); - - constructor() {} - - public registerView(view: T): void { - const databaseUri = view.databaseUri; - - if (this.views.has(databaseUri)) { - throw new Error(`View for database ${databaseUri} already registered`); - } - - this.views.set(databaseUri, view); - } - - public unregisterView(view: T): void { - this.views.delete(view.databaseUri); - } - - public getView(databaseUri: string): T | undefined { - return this.views.get(databaseUri); - } -} diff --git a/extensions/ql-vscode/src/model-editor/model-editor-view.ts b/extensions/ql-vscode/src/model-editor/model-editor-view.ts index e8e7e0904ce..c23f27173a5 100644 --- a/extensions/ql-vscode/src/model-editor/model-editor-view.ts +++ b/extensions/ql-vscode/src/model-editor/model-editor-view.ts @@ -44,7 +44,6 @@ import { import { AutoModeler } from "./auto-modeler"; import { telemetryListener } from "../common/vscode/telemetry"; import { ModelingStore } from "./modeling-store"; -import { ModelEditorViewTracker } from "./model-editor-view-tracker"; import { ModelingEvents } from "./modeling-events"; import { getModelsAsDataLanguage, ModelsAsDataLanguage } from "./languages"; import { runGenerateQueries } from "./generate"; @@ -60,7 +59,6 @@ export class ModelEditorView extends AbstractWebview< protected readonly app: App, private readonly modelingStore: ModelingStore, private readonly modelingEvents: ModelingEvents, - private readonly viewTracker: ModelEditorViewTracker, private readonly modelConfig: ModelConfigListener, private readonly databaseManager: DatabaseManager, private readonly cliServer: CodeQLCliServer, @@ -79,8 +77,6 @@ export class ModelEditorView extends AbstractWebview< this.registerToModelingEvents(); this.registerToModelConfigEvents(); - this.viewTracker.registerView(this); - this.autoModeler = new AutoModeler( app, cliServer, @@ -166,7 +162,7 @@ export class ModelEditorView extends AbstractWebview< } protected onPanelDispose(): void { - this.viewTracker.unregisterView(this); + // Nothing to do } protected async onMessage(msg: FromModelEditorMessage): Promise { @@ -573,12 +569,9 @@ export class ModelEditorView extends AbstractWebview< return; } - let existingView = this.viewTracker.getView( - addedDatabase.databaseUri.toString(), - ); - if (existingView) { - await existingView.focusView(); - + const addedDbUri = addedDatabase.databaseUri.toString(); + if (this.modelingStore.isDbOpen(addedDbUri)) { + this.modelingEvents.fireFocusModelEditorEvent(addedDbUri); return; } @@ -596,12 +589,8 @@ export class ModelEditorView extends AbstractWebview< // Check again just before opening the editor to ensure no model editor has been opened between // our first check and now. - existingView = this.viewTracker.getView( - addedDatabase.databaseUri.toString(), - ); - if (existingView) { - await existingView.focusView(); - + if (this.modelingStore.isDbOpen(addedDbUri)) { + this.modelingEvents.fireFocusModelEditorEvent(addedDbUri); return; } @@ -609,7 +598,6 @@ export class ModelEditorView extends AbstractWebview< this.app, this.modelingStore, this.modelingEvents, - this.viewTracker, this.modelConfig, this.databaseManager, this.cliServer, @@ -742,6 +730,22 @@ export class ModelEditorView extends AbstractWebview< } }), ); + + this.push( + this.modelingEvents.onRevealInModelEditor(async (event) => { + if (event.dbUri === this.databaseItem.databaseUri.toString()) { + await this.revealMethod(event.method); + } + }), + ); + + this.push( + this.modelingEvents.onFocusModelEditor(async (event) => { + if (event.dbUri === this.databaseItem.databaseUri.toString()) { + await this.focusView(); + } + }), + ); } private registerToModelConfigEvents() { diff --git a/extensions/ql-vscode/src/model-editor/modeling-events.ts b/extensions/ql-vscode/src/model-editor/modeling-events.ts index 795a8d9d27f..685a5d5900d 100644 --- a/extensions/ql-vscode/src/model-editor/modeling-events.ts +++ b/extensions/ql-vscode/src/model-editor/modeling-events.ts @@ -48,6 +48,15 @@ interface InProgressMethodsChangedEvent { readonly methods: ReadonlySet; } +interface RevealInModelEditorEvent { + dbUri: string; + method: Method; +} + +interface FocusModelEditorEvent { + dbUri: string; +} + export class ModelingEvents extends DisposableObject { public readonly onActiveDbChanged: AppEvent; public readonly onDbOpened: AppEvent; @@ -59,6 +68,8 @@ export class ModelingEvents extends DisposableObject { public readonly onModifiedMethodsChanged: AppEvent; public readonly onSelectedMethodChanged: AppEvent; public readonly onInProgressMethodsChanged: AppEvent; + public readonly onRevealInModelEditor: AppEvent; + public readonly onFocusModelEditor: AppEvent; private readonly onActiveDbChangedEventEmitter: AppEventEmitter; private readonly onDbOpenedEventEmitter: AppEventEmitter; @@ -70,6 +81,8 @@ export class ModelingEvents extends DisposableObject { private readonly onModifiedMethodsChangedEventEmitter: AppEventEmitter; private readonly onSelectedMethodChangedEventEmitter: AppEventEmitter; private readonly onInProgressMethodsChangedEventEmitter: AppEventEmitter; + private readonly onRevealInModelEditorEventEmitter: AppEventEmitter; + private readonly onFocusModelEditorEventEmitter: AppEventEmitter; constructor(app: App) { super(); @@ -126,6 +139,16 @@ export class ModelingEvents extends DisposableObject { ); this.onInProgressMethodsChanged = this.onInProgressMethodsChangedEventEmitter.event; + + this.onRevealInModelEditorEventEmitter = this.push( + app.createEventEmitter(), + ); + this.onRevealInModelEditor = this.onRevealInModelEditorEventEmitter.event; + + this.onFocusModelEditorEventEmitter = this.push( + app.createEventEmitter(), + ); + this.onFocusModelEditor = this.onFocusModelEditorEventEmitter.event; } public fireActiveDbChangedEvent() { @@ -220,4 +243,17 @@ export class ModelingEvents extends DisposableObject { methods, }); } + + public fireRevealInModelEditorEvent(dbUri: string, method: Method) { + this.onRevealInModelEditorEventEmitter.fire({ + dbUri, + method, + }); + } + + public fireFocusModelEditorEvent(dbUri: string) { + this.onFocusModelEditorEventEmitter.fire({ + dbUri, + }); + } } diff --git a/extensions/ql-vscode/src/model-editor/modeling-store.ts b/extensions/ql-vscode/src/model-editor/modeling-store.ts index 537dd9b7214..4124c4af1c5 100644 --- a/extensions/ql-vscode/src/model-editor/modeling-store.ts +++ b/extensions/ql-vscode/src/model-editor/modeling-store.ts @@ -112,6 +112,10 @@ export class ModelingStore extends DisposableObject { return this.state.size > 0; } + public isDbOpen(dbUri: string): boolean { + return this.state.has(dbUri); + } + /** * Returns the method for the given database item and method signature. * Returns undefined if no method exists with that signature. diff --git a/extensions/ql-vscode/test/__mocks__/model-editor/modelEditorViewTrackerMock.ts b/extensions/ql-vscode/test/__mocks__/model-editor/modelEditorViewTrackerMock.ts deleted file mode 100644 index 9a4655c6df9..00000000000 --- a/extensions/ql-vscode/test/__mocks__/model-editor/modelEditorViewTrackerMock.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { mockedObject } from "../../vscode-tests/utils/mocking.helpers"; -import { ModelEditorViewTracker } from "../../../src/model-editor/model-editor-view-tracker"; -import { ModelEditorView } from "../../../src/model-editor/model-editor-view"; - -export function createMockModelEditorViewTracker({ - registerView = jest.fn(), - unregisterView = jest.fn(), - getView = jest.fn(), -}: { - registerView?: ModelEditorViewTracker["registerView"]; - unregisterView?: ModelEditorViewTracker["unregisterView"]; - getView?: ModelEditorViewTracker["getView"]; -} = {}): ModelEditorViewTracker { - return mockedObject>({ - registerView, - unregisterView, - getView, - }); -} diff --git a/extensions/ql-vscode/test/__mocks__/model-editor/modelingEventsMock.ts b/extensions/ql-vscode/test/__mocks__/model-editor/modelingEventsMock.ts index f297ca9d456..4d1a4f47c28 100644 --- a/extensions/ql-vscode/test/__mocks__/model-editor/modelingEventsMock.ts +++ b/extensions/ql-vscode/test/__mocks__/model-editor/modelingEventsMock.ts @@ -10,6 +10,8 @@ export function createMockModelingEvents({ onModeledMethodsChanged = jest.fn(), onModifiedMethodsChanged = jest.fn(), onInProgressMethodsChanged = jest.fn(), + onRevealInModelEditor = jest.fn(), + onFocusModelEditor = jest.fn(), }: { onActiveDbChanged?: ModelingEvents["onActiveDbChanged"]; onDbClosed?: ModelingEvents["onDbClosed"]; @@ -19,6 +21,8 @@ export function createMockModelingEvents({ onModeledMethodsChanged?: ModelingEvents["onModeledMethodsChanged"]; onModifiedMethodsChanged?: ModelingEvents["onModifiedMethodsChanged"]; onInProgressMethodsChanged?: ModelingEvents["onInProgressMethodsChanged"]; + onRevealInModelEditor?: ModelingEvents["onRevealInModelEditor"]; + onFocusModelEditor?: ModelingEvents["onFocusModelEditor"]; } = {}): ModelingEvents { return mockedObject({ onActiveDbChanged, @@ -29,5 +33,7 @@ export function createMockModelingEvents({ onModeledMethodsChanged, onModifiedMethodsChanged, onInProgressMethodsChanged, + onRevealInModelEditor, + onFocusModelEditor, }); } diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-view.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-view.test.ts index e9a4e3d8ecc..046ff196fd1 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-view.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-view.test.ts @@ -9,7 +9,6 @@ import { mockEmptyDatabaseManager } from "../query-testing/test-runner-helpers"; import { QueryRunner } from "../../../../src/query-server"; import { ExtensionPack } from "../../../../src/model-editor/shared/extension-pack"; import { createMockModelingStore } from "../../../__mocks__/model-editor/modelingStoreMock"; -import { createMockModelEditorViewTracker } from "../../../__mocks__/model-editor/modelEditorViewTrackerMock"; import { ModelConfigListener } from "../../../../src/config"; import { createMockModelingEvents } from "../../../__mocks__/model-editor/modelingEventsMock"; import { QueryLanguage } from "../../../../src/common/query-language"; @@ -18,7 +17,6 @@ describe("ModelEditorView", () => { const app = createMockApp({}); const modelingStore = createMockModelingStore(); const modelingEvents = createMockModelingEvents(); - const viewTracker = createMockModelEditorViewTracker(); const modelConfig = mockedObject({ onDidChangeConfiguration: jest.fn(), }); @@ -48,7 +46,6 @@ describe("ModelEditorView", () => { app, modelingStore, modelingEvents, - viewTracker, modelConfig, databaseManager, cliServer, From 144033967dc12845ccd5c9cde491131b70ef1bbd Mon Sep 17 00:00:00 2001 From: Shati Patel <42641846+shati-patel@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:44:46 +0000 Subject: [PATCH 07/11] Rename "allowHttp" setting (#3069) --- extensions/ql-vscode/CHANGELOG.md | 2 +- extensions/ql-vscode/src/config.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index 6018434b834..e9feeb8d526 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -5,7 +5,7 @@ - When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047) - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view. - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`. -- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) +- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) & [#3069](https://github.com/github/vscode-codeql/pull/3069) - The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055) ## 1.9.4 - 6 November 2023 diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts index a13e1f50ad8..f8d44dc4070 100644 --- a/extensions/ql-vscode/src/config.ts +++ b/extensions/ql-vscode/src/config.ts @@ -641,12 +641,23 @@ export function isCodespacesTemplate() { return !!CODESPACES_TEMPLATE.getValue(); } +// Deprecated after v1.9.4. Can be removed in a few versions. +const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING); +const DEPRECATED_ALLOW_HTTP_SETTING = new Setting( + "allowHttp", + DATABASE_DOWNLOAD_SETTING, +); + const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING); const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING); export function allowHttp(): boolean { - return ALLOW_HTTP_SETTING.getValue() || false; + return ( + ALLOW_HTTP_SETTING.getValue() || + DEPRECATED_ALLOW_HTTP_SETTING.getValue() || + false + ); } const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting( From a271f7b36e1a560c175ba0172416517e000f43dc Mon Sep 17 00:00:00 2001 From: Shati Patel <42641846+shati-patel@users.noreply.github.com> Date: Tue, 14 Nov 2023 13:29:41 +0000 Subject: [PATCH 08/11] Fix minor bugs with queries panel + language selector (#3070) --- .../src/databases/local-databases/database-manager.ts | 2 +- extensions/ql-vscode/src/local-queries/qlpack-generator.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index c4d3cde1146..9dd08d59d60 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -116,7 +116,7 @@ export class DatabaseManager extends DisposableObject { this.languageContext.onLanguageContextChanged(async () => { if ( this.currentDatabaseItem !== undefined && - !this.languageContext.isSelectedLanguage( + !this.languageContext.shouldInclude( tryGetQueryLanguage(this.currentDatabaseItem.language), ) ) { diff --git a/extensions/ql-vscode/src/local-queries/qlpack-generator.ts b/extensions/ql-vscode/src/local-queries/qlpack-generator.ts index 55b649a9c41..ed7fbef414c 100644 --- a/extensions/ql-vscode/src/local-queries/qlpack-generator.ts +++ b/extensions/ql-vscode/src/local-queries/qlpack-generator.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from "fs-extra"; +import { ensureDir, writeFile } from "fs-extra"; import { dump } from "js-yaml"; import { dirname, join } from "path"; import { Uri } from "vscode"; @@ -76,7 +76,7 @@ export class QlPackGenerator { } private async createWorkspaceFolder() { - await mkdir(this.folderUri.fsPath); + await ensureDir(this.folderUri.fsPath); } private async createQlPackYaml() { From 7fe707a42d1fe2a868aefef3145d01c3aba6205d Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 15 Nov 2023 14:54:00 +0100 Subject: [PATCH 09/11] Add database id and created_at to origin --- extensions/ql-vscode/src/databases/database-fetcher.ts | 9 ++++++++- .../src/databases/local-databases/database-origin.ts | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index 6fa407f7469..c71ebf41579 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -204,7 +204,8 @@ export async function downloadGitHubDatabase( return; } - const { databaseUrl, name, owner, commitOid } = result; + const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } = + result; /** * The 'token' property of the token object returned by `octokit.auth()`. @@ -229,6 +230,8 @@ export async function downloadGitHubDatabase( { type: "github", repository: nwo, + databaseId, + databaseCreatedAt, commitOid, }, progress, @@ -550,6 +553,8 @@ export async function convertGithubNwoToDatabaseUrl( databaseUrl: string; owner: string; name: string; + databaseId: number; + databaseCreatedAt: string; commitOid: string | null; } | undefined @@ -582,6 +587,8 @@ export async function convertGithubNwoToDatabaseUrl( databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`, owner, name: repo, + databaseId: databaseForLanguage.id, + databaseCreatedAt: databaseForLanguage.created_at, commitOid: databaseForLanguage.commit_oid, }; } catch (e) { diff --git a/extensions/ql-vscode/src/databases/local-databases/database-origin.ts b/extensions/ql-vscode/src/databases/local-databases/database-origin.ts index 2b397f4277d..7c9aba02d52 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-origin.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-origin.ts @@ -10,6 +10,8 @@ interface DatabaseOriginArchive { interface DatabaseOriginGitHub { type: "github"; repository: string; + databaseId: number; + databaseCreatedAt: string; commitOid: string | null; } From f8fa863b9326c1d4138a21309a41e08471476ef9 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 15 Nov 2023 17:11:24 +0100 Subject: [PATCH 10/11] Use Octokit REST for listing CodeQL databases This switches the request to the GitHub API for listing CodeQL databases from a custom request to the Octokit REST API. This allows us to be more type-safe without introducing our own types. The update to `@octokit/openapi-types` was necessary to have access to the `commit_oid` field. --- extensions/ql-vscode/package-lock.json | 12 +++---- .../src/databases/database-fetcher.ts | 12 +++---- .../databases/database-fetcher.test.ts | 35 ++++++++++++------- .../vscode-tests/utils/mocking.helpers.ts | 9 +++++ 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index 02f8c984962..0000501907f 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -4392,9 +4392,9 @@ } }, "node_modules/@octokit/openapi-types": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz", - "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw==" + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.2.tgz", + "integrity": "sha512-8li32fUDUeml/ACRp/njCWTsk5t17cfTM1jp9n08pBrqs5cDFJubtjsSnuz56r5Tad6jdEPJld7LxNp9dNcyjQ==" }, "node_modules/@octokit/plugin-paginate-rest": { "version": "9.0.0", @@ -35269,9 +35269,9 @@ } }, "@octokit/openapi-types": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz", - "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw==" + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.2.tgz", + "integrity": "sha512-8li32fUDUeml/ACRp/njCWTsk5t17cfTM1jp9n08pBrqs5cDFJubtjsSnuz56r5Tad6jdEPJld7LxNp9dNcyjQ==" }, "@octokit/plugin-paginate-rest": { "version": "9.0.0", diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index c71ebf41579..3a8e2290341 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -562,10 +562,10 @@ export async function convertGithubNwoToDatabaseUrl( try { const [owner, repo] = nwo.split("/"); - const response = await octokit.request( - "GET /repos/:owner/:repo/code-scanning/codeql/databases", - { owner, repo }, - ); + const response = await octokit.rest.codeScanning.listCodeqlDatabases({ + owner, + repo, + }); const languages = response.data.map((db: any) => db.language); @@ -584,12 +584,12 @@ export async function convertGithubNwoToDatabaseUrl( } return { - databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`, + databaseUrl: databaseForLanguage.url, owner, name: repo, databaseId: databaseForLanguage.id, databaseCreatedAt: databaseForLanguage.created_at, - commitOid: databaseForLanguage.commit_oid, + commitOid: databaseForLanguage.commit_oid ?? null, }; } catch (e) { void extLogger.log(`Error: ${getErrorMessage(e)}`); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/database-fetcher.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/database-fetcher.test.ts index f7f8e7b7046..71ac166734d 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/database-fetcher.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/database-fetcher.test.ts @@ -8,7 +8,11 @@ import { findDirWithFile, } from "../../../../src/databases/database-fetcher"; import * as Octokit from "@octokit/rest"; -import { mockedQuickPickItem } from "../../utils/mocking.helpers"; +import { + mockedObject, + mockedOctokitFunction, + mockedQuickPickItem, +} from "../../utils/mocking.helpers"; // These tests make API calls and may need extra time to complete. jest.setTimeout(10000); @@ -18,10 +22,17 @@ describe("database-fetcher", () => { let quickPickSpy: jest.SpiedFunction; const progressSpy = jest.fn(); - const mockRequest = jest.fn(); - const octokit: Octokit.Octokit = { - request: mockRequest, - } as unknown as Octokit.Octokit; + const mockListCodeqlDatabases = mockedOctokitFunction< + "codeScanning", + "listCodeqlDatabases" + >(); + const octokit = mockedObject({ + rest: { + codeScanning: { + listCodeqlDatabases: mockListCodeqlDatabases, + }, + }, + }); // We can't make the real octokit request (since we need credentials), so we mock the response. const successfullMockApiResponse = { @@ -72,7 +83,7 @@ describe("database-fetcher", () => { }); it("should convert a GitHub nwo to a database url", async () => { - mockRequest.mockResolvedValue(successfullMockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(successfullMockApiResponse); quickPickSpy.mockResolvedValue( mockedQuickPickItem({ label: "JavaScript", @@ -93,7 +104,7 @@ describe("database-fetcher", () => { const { databaseUrl, name, owner } = result; expect(databaseUrl).toBe( - "https://api.github.com/repos/github/codeql/code-scanning/codeql/databases/javascript", + "https://api.github.com/repositories/143040428/code-scanning/codeql/databases/javascript", ); expect(name).toBe("codeql"); expect(owner).toBe("github"); @@ -128,7 +139,7 @@ describe("database-fetcher", () => { }, status: 404, }; - mockRequest.mockResolvedValue(mockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(mockApiResponse); const githubRepo = "foo/bar-not-real"; await expect( convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy), @@ -142,7 +153,7 @@ describe("database-fetcher", () => { data: [], }; - mockRequest.mockResolvedValue(mockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(mockApiResponse); const githubRepo = "foo/bar-with-no-dbs"; await expect( convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy), @@ -153,7 +164,7 @@ describe("database-fetcher", () => { describe("when language is already provided", () => { describe("when language is valid", () => { it("should not prompt the user", async () => { - mockRequest.mockResolvedValue(successfullMockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(successfullMockApiResponse); const githubRepo = "github/codeql"; await convertGithubNwoToDatabaseUrl( githubRepo, @@ -167,7 +178,7 @@ describe("database-fetcher", () => { describe("when language is invalid", () => { it("should prompt for language", async () => { - mockRequest.mockResolvedValue(successfullMockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(successfullMockApiResponse); const githubRepo = "github/codeql"; await convertGithubNwoToDatabaseUrl( githubRepo, @@ -182,7 +193,7 @@ describe("database-fetcher", () => { describe("when language is not provided", () => { it("should prompt for language", async () => { - mockRequest.mockResolvedValue(successfullMockApiResponse); + mockListCodeqlDatabases.mockResolvedValue(successfullMockApiResponse); const githubRepo = "github/codeql"; await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy); expect(quickPickSpy).toHaveBeenCalled(); diff --git a/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts b/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts index 388e048b17f..bb826293ded 100644 --- a/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts +++ b/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts @@ -1,5 +1,6 @@ import type { QuickPickItem, window, Uri } from "vscode"; import { DatabaseItem } from "../../../src/databases/local-databases"; +import * as Octokit from "@octokit/rest"; export type DeepPartial = T extends object ? { @@ -57,6 +58,14 @@ export function mockedObject( }); } +export function mockedOctokitFunction< + Namespace extends keyof Octokit.Octokit["rest"], + Name extends keyof Octokit.Octokit["rest"][Namespace], +>(): Octokit.Octokit["rest"][Namespace][Name] & jest.Mock { + const fn = jest.fn(); + return fn as unknown as Octokit.Octokit["rest"][Namespace][Name] & jest.Mock; +} + export function mockDatabaseItem( props: DeepPartial = {}, ): DatabaseItem { From d3608159b89daf24789ff9e115bc0a5b2a09c787 Mon Sep 17 00:00:00 2001 From: Shati Patel <42641846+shati-patel@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:55:24 +0000 Subject: [PATCH 11/11] Remove feature flags for queries panel + language selector (#3075) --- extensions/ql-vscode/CHANGELOG.md | 5 +++++ extensions/ql-vscode/package.json | 10 ++-------- extensions/ql-vscode/src/config.ts | 9 --------- .../ql-vscode/src/queries-panel/queries-module.ts | 6 ------ 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index e9feeb8d526..3f8933fbf92 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,6 +2,11 @@ ## [UNRELEASED] +- Add new CodeQL views for managing databases and queries: + 1. A queries panel, to create and run queries in one place. + 2. A language selector, to quickly determine the language compatibility of databases and queries. + + For more information, see the [documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#filtering-databases-and-queries-by-language). - When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047) - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view. - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`. diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index a440f7139a3..12d670a4d81 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1686,10 +1686,6 @@ "command": "codeQL.mockGitHubApiServer.unloadScenario", "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded" }, - { - "command": "codeQL.createQuery", - "when": "config.codeQL.codespacesTemplate || config.codeQL.canary && config.codeQL.queriesPanel" - }, { "command": "codeQLTests.acceptOutputContextTestItem", "when": "false" @@ -1774,8 +1770,7 @@ "ql-container": [ { "id": "codeQLLanguageSelection", - "name": "Language", - "when": "config.codeQL.canary && config.codeQL.showLanguageFilter" + "name": "Language" }, { "id": "codeQLDatabases", @@ -1783,8 +1778,7 @@ }, { "id": "codeQLQueries", - "name": "Queries", - "when": "config.codeQL.canary && config.codeQL.queriesPanel" + "name": "Queries" }, { "id": "codeQLVariantAnalysisRepositories", diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts index f8d44dc4070..f672f0ed271 100644 --- a/extensions/ql-vscode/src/config.ts +++ b/extensions/ql-vscode/src/config.ts @@ -710,15 +710,6 @@ export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) { ); } -/** - * A flag indicating whether to show the queries panel in the QL view container. - */ -const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING); - -export function showQueriesPanel(): boolean { - return !!QUERIES_PANEL.getValue(); -} - const MODEL_SETTING = new Setting("model", ROOT_SETTING); const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING); const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING); diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts index 7608b90658d..d09290a57a5 100644 --- a/extensions/ql-vscode/src/queries-panel/queries-module.ts +++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts @@ -1,7 +1,6 @@ import { CodeQLCliServer } from "../codeql-cli/cli"; import { extLogger } from "../common/logging/vscode"; import { App } from "../common/app"; -import { isCanary, showQueriesPanel } from "../config"; import { DisposableObject } from "../common/disposable-object"; import { QueriesPanel } from "./queries-panel"; import { QueryDiscovery } from "./query-discovery"; @@ -41,11 +40,6 @@ export class QueriesModule extends DisposableObject { langauageContext: LanguageContextStore, cliServer: CodeQLCliServer, ): void { - // Currently, we only want to expose the new panel when we are in canary mode - // and the user has enabled the "Show queries panel" flag. - if (!isCanary() || !showQueriesPanel()) { - return; - } void extLogger.log("Initializing queries panel."); const queryPackDiscovery = new QueryPackDiscovery(cliServer);