Merge remote-tracking branch 'origin/main' into koesie10/find-github-…

…repository

extensions/ql-vscode/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [UNRELEASED]
 
+- Add new CodeQL views for managing databases and queries:
+  1. A queries panel, to create and run queries in one place.
+  2. A language selector, to quickly determine the language compatibility of databases and queries.
+
+  For more information, see the [documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#filtering-databases-and-queries-by-language).
+- When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047)
+  - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view.
+  - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`.
+- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) & [#3069](https://github.com/github/vscode-codeql/pull/3069)
 - The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055)
 
 ## 1.9.4 - 6 November 2023

extensions/ql-vscode/package.json

@@ -374,13 +374,23 @@
       },
       {
         "type": "object",
-        "title": "Downloading databases",
+        "title": "Adding databases",
         "order": 6,
         "properties": {
+          "codeQL.addingDatabases.allowHttp": {
+            "type": "boolean",
+            "default": false,
+            "description": "Allow databases to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+          },
           "codeQL.databaseDownload.allowHttp": {
+            "type": "boolean",
+            "markdownDeprecationMessage": "**Deprecated**: Please use `#codeQL.addingDatabases.allowHttp#` instead.",
+            "deprecationMessage": "Deprecated: Please use codeQL.addingDatabases.allowHttp instead."
+          },
+          "codeQL.addingDatabases.addDatabaseSourceToWorkspace": {
             "type": "boolean",
             "default": false,
-            "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+            "markdownDescription": "When adding a CodeQL database, automatically add the database's source folder as a workspace folder. Warning: enabling this option in a single-folder workspace will cause the workspace to reload as a [multi-root workspace](https://code.visualstudio.com/docs/editor/multi-root-workspaces). This may cause query history and database lists to be reset."
           }
         }
       },
@@ -1678,10 +1688,6 @@
           "command": "codeQL.mockGitHubApiServer.unloadScenario",
           "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded"
         },
-        {
-          "command": "codeQL.createQuery",
-          "when": "config.codeQL.codespacesTemplate || config.codeQL.canary && config.codeQL.queriesPanel"
-        },
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
@@ -1766,17 +1772,15 @@
       "ql-container": [
         {
           "id": "codeQLLanguageSelection",
-          "name": "Language",
-          "when": "config.codeQL.canary && config.codeQL.showLanguageFilter"
+          "name": "Language"
         },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
         },
         {
           "id": "codeQLQueries",
-          "name": "Queries",
-          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+          "name": "Queries"
         },
         {
           "id": "codeQLVariantAnalysisRepositories",

extensions/ql-vscode/src/config.ts

@@ -641,12 +641,32 @@ export function isCodespacesTemplate() {
   return !!CODESPACES_TEMPLATE.getValue<boolean>();
 }
 
+// Deprecated after v1.9.4. Can be removed in a few versions.
 const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
+const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
+  "allowHttp",
+  DATABASE_DOWNLOAD_SETTING,
+);
+
+const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
 
-const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING);
+const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
 export function allowHttp(): boolean {
-  return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
+  return (
+    ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    DEPRECATED_ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    false
+  );
+}
+
+const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting(
+  "addDatabaseSourceToWorkspace",
+  ADDING_DATABASES_SETTING,
+);
+
+export function addDatabaseSourceToWorkspace(): boolean {
+  return ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING.getValue<boolean>() || false;
 }
 
 /**
@@ -690,15 +710,6 @@ export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
   );
 }
 
-/**
- * A flag indicating whether to show the queries panel in the QL view container.
- */
-const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
-
-export function showQueriesPanel(): boolean {
-  return !!QUERIES_PANEL.getValue<boolean>();
-}
-
 const MODEL_SETTING = new Setting("model", ROOT_SETTING);
 const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);

extensions/ql-vscode/src/databases/config/db-config.ts

@@ -1,6 +1,8 @@
 // Contains models and consts for the data we want to store in the database config.
 // Changes to these models should be done carefully and account for backwards compatibility of data.
 
+import { DatabaseOrigin } from "../local-databases/database-origin";
+
 export const DB_CONFIG_VERSION = 1;
 
 export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
   name: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
 }
 

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -29,10 +29,11 @@ import {
 } from "../common/github-url-identifier-helper";
 import { Credentials } from "../common/authentication";
 import { AppCommandManager } from "../common/commands";
-import { allowHttp } from "../config";
+import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
 import { getLanguageDisplayName } from "../common/query-language";
+import { DatabaseOrigin } from "./local-databases/database-origin";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -62,6 +63,10 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     undefined,
+    {
+      type: "url",
+      url: databaseUrl,
+    },
     progress,
     cli,
   );
@@ -99,7 +104,7 @@ export async function promptImportGithubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const githubRepo = await askForGitHubRepo(progress);
   if (!githubRepo) {
@@ -178,7 +183,7 @@ export async function downloadGitHubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
   if (!isValidGitHubNwo(nwo)) {
@@ -199,7 +204,8 @@ export async function downloadGitHubDatabase(
     return;
   }
 
-  const { databaseUrl, name, owner } = result;
+  const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
+    result;
 
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
@@ -221,6 +227,13 @@ export async function downloadGitHubDatabase(
     databaseManager,
     storagePath,
     `${owner}/${name}`,
+    {
+      type: "github",
+      repository: nwo,
+      databaseId,
+      databaseCreatedAt,
+      commitOid,
+    },
     progress,
     cli,
     makeSelected,
@@ -250,6 +263,10 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       undefined,
+      {
+        type: "archive",
+        path: databaseUrl,
+      },
       progress,
       cli,
     );
@@ -282,6 +299,7 @@ export async function importArchiveDatabase(
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
+ * @param origin the origin of the database
  * @param progress callback to send progress messages to
  * @param makeSelected make the new database selected in the databases panel (default: true)
  * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -292,10 +310,11 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   nameOverride: string | undefined,
+  origin: DatabaseOrigin,
   progress: ProgressCallback,
   cli?: CodeQLCliServer,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem> {
   progress({
     message: "Getting database",
@@ -336,6 +355,7 @@ async function databaseArchiveFetcher(
 
     const item = await databaseManager.openDatabase(
       Uri.file(dbPath),
+      origin,
       makeSelected,
       nameOverride,
       {
@@ -476,7 +496,7 @@ async function checkForFailingResponse(
     return response;
   }
 
-  // An error downloading the database. Attempt to extract the resaon behind it.
+  // An error downloading the database. Attempt to extract the reason behind it.
   const text = await response.text();
   let msg: string;
   try {
@@ -533,16 +553,19 @@ export async function convertGithubNwoToDatabaseUrl(
       databaseUrl: string;
       owner: string;
       name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
     }
   | undefined
 > {
   try {
     const [owner, repo] = nwo.split("/");
 
-    const response = await octokit.request(
-      "GET /repos/:owner/:repo/code-scanning/codeql/databases",
-      { owner, repo },
-    );
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+      owner,
+      repo,
+    });
 
     const languages = response.data.map((db: any) => db.language);
 
@@ -553,10 +576,20 @@ export async function convertGithubNwoToDatabaseUrl(
       }
     }
 
+    const databaseForLanguage = response.data.find(
+      (db: any) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
     return {
-      databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
+      databaseUrl: databaseForLanguage.url,
       owner,
       name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid ?? null,
     };
   } catch (e) {
     void extLogger.log(`Error: ${getErrorMessage(e)}`);

extensions/ql-vscode/src/databases/db-item.ts

@@ -1,5 +1,7 @@
 // This file contains models that are used to represent the databases.
 
+import { DatabaseOrigin } from "./local-databases/database-origin";
+
 export enum DbItemKind {
   RootLocal = "RootLocal",
   LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
   databaseName: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
   parentListName?: string;
 }

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -197,6 +197,7 @@ function createLocalDb(
     databaseName: db.name,
     dateAdded: db.dateAdded,
     language: db.language,
+    origin: db.origin,
     storagePath: db.storagePath,
     selected: !!selected,
     parentListName: listName,

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
 
               await this.databaseManager.openDatabase(
                 uri,
+                {
+                  type: "folder",
+                },
                 makeSelected,
                 nameOverride,
                 {
@@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(uri);
+            await this.databaseManager.openDatabase(uri, {
+              type: "folder",
+            });
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri);
+          return await this.databaseManager.openDatabase(fixedUri, {
+            type: "folder",
+          });
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.