Merge pull request #3066 from github/koesie10/add-database-source

Add origin metadata to database items

Author: koesie10

extensions/ql-vscode/src/databases/config/db-config.ts

@@ -1,6 +1,8 @@
 // Contains models and consts for the data we want to store in the database config.
 // Changes to these models should be done carefully and account for backwards compatibility of data.
 
+import { DatabaseOrigin } from "../local-databases/database-origin";
+
 export const DB_CONFIG_VERSION = 1;
 
 export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
   name: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
 }
 

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -33,6 +33,7 @@ import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
 import { getLanguageDisplayName } from "../common/query-language";
+import { DatabaseOrigin } from "./local-databases/database-origin";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -62,6 +63,10 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     undefined,
+    {
+      type: "url",
+      url: databaseUrl,
+    },
     progress,
     cli,
   );
@@ -199,7 +204,8 @@ export async function downloadGitHubDatabase(
     return;
   }
 
-  const { databaseUrl, name, owner } = result;
+  const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
+    result;
 
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
@@ -221,6 +227,13 @@ export async function downloadGitHubDatabase(
     databaseManager,
     storagePath,
     `${owner}/${name}`,
+    {
+      type: "github",
+      repository: nwo,
+      databaseId,
+      databaseCreatedAt,
+      commitOid,
+    },
     progress,
     cli,
     makeSelected,
@@ -250,6 +263,10 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       undefined,
+      {
+        type: "archive",
+        path: databaseUrl,
+      },
       progress,
       cli,
     );
@@ -282,6 +299,7 @@ export async function importArchiveDatabase(
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
+ * @param origin the origin of the database
  * @param progress callback to send progress messages to
  * @param makeSelected make the new database selected in the databases panel (default: true)
  * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -292,6 +310,7 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   nameOverride: string | undefined,
+  origin: DatabaseOrigin,
   progress: ProgressCallback,
   cli?: CodeQLCliServer,
   makeSelected = true,
@@ -336,6 +355,7 @@ async function databaseArchiveFetcher(
 
     const item = await databaseManager.openDatabase(
       Uri.file(dbPath),
+      origin,
       makeSelected,
       nameOverride,
       {
@@ -533,6 +553,9 @@ export async function convertGithubNwoToDatabaseUrl(
       databaseUrl: string;
       owner: string;
       name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
     }
   | undefined
 > {
@@ -553,10 +576,20 @@ export async function convertGithubNwoToDatabaseUrl(
       }
     }
 
+    const databaseForLanguage = response.data.find(
+      (db: any) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
     return {
       databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
       owner,
       name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid,
     };
   } catch (e) {
     void extLogger.log(`Error: ${getErrorMessage(e)}`);

extensions/ql-vscode/src/databases/db-item.ts

@@ -1,5 +1,7 @@
 // This file contains models that are used to represent the databases.
 
+import { DatabaseOrigin } from "./local-databases/database-origin";
+
 export enum DbItemKind {
   RootLocal = "RootLocal",
   LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
   databaseName: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
   parentListName?: string;
 }

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -197,6 +197,7 @@ function createLocalDb(
     databaseName: db.name,
     dateAdded: db.dateAdded,
     language: db.language,
+    origin: db.origin,
     storagePath: db.storagePath,
     selected: !!selected,
     parentListName: listName,

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
 
               await this.databaseManager.openDatabase(
                 uri,
+                {
+                  type: "folder",
+                },
                 makeSelected,
                 nameOverride,
                 {
@@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(uri);
+            await this.databaseManager.openDatabase(uri, {
+              type: "folder",
+            });
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri);
+          return await this.databaseManager.openDatabase(fixedUri, {
+            type: "folder",
+          });
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
 import { containsPath, pathsEqual } from "../../common/files";
 import { DatabaseContents } from "./database-contents";
+import { DatabaseOrigin } from "./database-origin";
 
 export class DatabaseItemImpl implements DatabaseItem {
   // These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.dateAdded;
   }
 
+  public get origin(): DatabaseOrigin | undefined {
+    return this.options.origin;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -2,6 +2,7 @@ import vscode from "vscode";
 import * as cli from "../../codeql-cli/cli";
 import { DatabaseContents } from "./database-contents";
 import { DatabaseOptions } from "./database-options";
+import { DatabaseOrigin } from "./database-origin";
 
 /** An item in the list of available databases */
 export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
    */
   readonly dateAdded: number | undefined;
 
+  /**
+   * The origin this database item was retrieved from or undefined if unknown.
+   */
+  readonly origin: DatabaseOrigin | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -35,6 +35,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import { LanguageContextStore } from "../../language-context-store";
+import { DatabaseOrigin } from "./database-origin";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -132,14 +133,19 @@ export class DatabaseManager extends DisposableObject {
    */
   public async openDatabase(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     makeSelected = true,
     displayName?: string,
     {
       isTutorialDatabase = false,
       addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
     }: OpenDatabaseOptions = {},
   ): Promise<DatabaseItem> {
-    const databaseItem = await this.createDatabaseItem(uri, displayName);
+    const databaseItem = await this.createDatabaseItem(
+      uri,
+      origin,
+      displayName,
+    );
 
     return await this.addExistingDatabaseItem(
       databaseItem,
@@ -190,6 +196,7 @@ export class DatabaseManager extends DisposableObject {
    */
   private async createDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     displayName: string | undefined,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -198,6 +205,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
+      origin,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -213,6 +221,7 @@ export class DatabaseManager extends DisposableObject {
    */
   public async createOrOpenDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(uri);
     if (existingItem !== undefined) {
@@ -221,7 +230,7 @@ export class DatabaseManager extends DisposableObject {
     }
 
     // We don't add this to the list automatically, but the user can add it later.
-    return this.createDatabaseItem(uri, undefined);
+    return this.createDatabaseItem(uri, origin, undefined);
   }
 
   public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -356,6 +365,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let dateAdded = undefined;
     let language = undefined;
+    let origin = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -364,6 +374,7 @@ export class DatabaseManager extends DisposableObject {
         dateAdded = state.options.dateAdded;
       }
       language = state.options.language;
+      origin = state.options.origin;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -376,6 +387,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded,
       language,
+      origin,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -1,10 +1,14 @@
+import { DatabaseOrigin } from "./database-origin";
+
 export interface DatabaseOptions {
   displayName?: string;
   dateAdded?: number | undefined;
   language?: string;
+  origin?: DatabaseOrigin;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
+  origin: DatabaseOrigin | undefined;
 }

extensions/ql-vscode/src/databases/local-databases/database-origin.ts

@@ -0,0 +1,32 @@
+interface DatabaseOriginFolder {
+  type: "folder";
+}
+
+interface DatabaseOriginArchive {
+  type: "archive";
+  path: string;
+}
+
+interface DatabaseOriginGitHub {
+  type: "github";
+  repository: string;
+  databaseId: number;
+  databaseCreatedAt: string;
+  commitOid: string | null;
+}
+
+interface DatabaseOriginInternet {
+  type: "url";
+  url: string;
+}
+
+interface DatabaseOriginDebugger {
+  type: "debugger";
+}
+
+export type DatabaseOrigin =
+  | DatabaseOriginFolder
+  | DatabaseOriginArchive
+  | DatabaseOriginGitHub
+  | DatabaseOriginInternet
+  | DatabaseOriginDebugger;