wip: Search AppMaps for scope and stack

packages/cli/src/cli.ts

@@ -24,6 +24,7 @@ const InventoryCommand = require('./inventoryCommand');
 const OpenCommand = require('./cmds/open/open');
 const InspectCommand = require('./cmds/inspect/inspect');
 const RecordCommand = require('./cmds/record/record');
+const SearchCommand = require('./cmds/search/search');
 import InstallCommand from './cmds/agentInstaller/install-agent';
 import StatusCommand from './cmds/agentInstaller/status';
 import PruneCommand from './cmds/prune/prune';
@@ -451,6 +452,7 @@ yargs(process.argv.slice(2))
   .command(RecordCommand)
   .command(StatusCommand)
   .command(InspectCommand)
+  .command(SearchCommand)
   .command(PruneCommand)
   .strict()
   .demandCommand()

packages/cli/src/cmds/search/sanitizeStack.ts

@@ -0,0 +1,25 @@
+import isNumeric from "../../lib/isNumeric";
+
+/**
+ * Process and sanitize a raw user-input stack trace. The output is an array of strings, each of the
+ * form `path/to/file.ext(:<lineno>)?`, where lineno is an optional line number. The first (0th)
+ * entry in the input stack is expected to be the deepest function. The output stack lines are in
+ * the reverse order.
+ *
+ * @param stack raw stack trace input from the user.
+ */
+export function sanitizeStack(stack: string): string[] {
+  const sanitize = (line: string): string => {
+    const [path, lineno] = line.split(':', 2);
+    const result = [path];
+    if (isNumeric(lineno)) result.push(lineno);
+    return result.join(':');
+  };
+
+  return stack
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line !== '')
+    .map(sanitize)
+    .reverse();
+}

packages/cli/src/cmds/search/search.ts

@@ -0,0 +1,106 @@
+import yargs, { number } from 'yargs';
+import readline from 'readline';
+import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory';
+import { locateAppMapDir } from '../../lib/locateAppMapDir';
+import FindCodeObjects from '../../search/findCodeObjects';
+import FindEvents from '../../search/findEvents';
+import { verbose } from '../../utils';
+import FindStack, { FindStackMatch } from '../../search/findStack';
+import { sanitizeStack } from './sanitizeStack';
+
+export const command = 'search';
+export const describe =
+  'Search AppMaps for references to a code objects (package, function, line, class, query, route, etc)';
+
+export const builder = (args) => {
+  args.option('directory', {
+    describe: 'program working directory',
+    type: 'string',
+    alias: 'd',
+  });
+  args.option('appmap-dir', {
+    describe: 'directory to recursively inspect for AppMaps',
+  });
+  args.option('route', {
+    describe: 'a route which all matches must contain',
+  });
+  args.option('limit', {
+    describe: 'number of top matches to print',
+    type: number,
+    default: 20,
+  });
+  return args.strict();
+};
+
+export const handler = async (argv) => {
+  verbose(argv.verbose);
+  handleWorkingDirectory(argv.directory);
+  const appmapDir = await locateAppMapDir(argv.appmapDir);
+  const route = argv.route;
+  const limit = argv.limit;
+
+  if (!route) yargs.exit(1, new Error(`No route was provided`));
+
+  const routeParam = `route:${route}`;
+  let stack: string;
+
+  if (process.stdin.isTTY) {
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+    stack = await new Promise((resolve) => {
+      rl.question(`Enter a stack trace to search for: `, resolve);
+    });
+  } else {
+    const result: Buffer[] = [];
+    let length = 0;
+    for await (const chunk of process.stdin) {
+      result.push(chunk);
+      length += chunk.length;
+    }
+    stack = Buffer.concat(result, length).toString('utf8');
+  }
+
+  if (!stack) yargs.exit(1, new Error(`No stack was provided`));
+  const stackLines = sanitizeStack(stack);
+
+  const finder = new FindCodeObjects(appmapDir, routeParam);
+  const codeObjectMatches = await finder.find(
+    (count) => {},
+    () => {}
+  );
+
+  if (codeObjectMatches?.length === 0) {
+    return yargs.exit(1, new Error(`Code object '${routeParam}' not found`));
+  }
+
+  const result: FindStackMatch[] = [];
+  await Promise.all(
+    codeObjectMatches.map(async (codeObjectMatch) => {
+      const findStack = new FindStack(codeObjectMatch.appmap, stackLines);
+      const matches = await findStack.match();
+      result.push(...matches);
+    })
+  );
+
+  let duplicateCount = 0;
+  const hashes = new Set<string>();
+  result
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+    .forEach((match) => {
+      if (hashes.has(match.hash_v2)) {
+        duplicateCount += 1;
+        return;
+      }
+      hashes.add(match.hash_v2);
+      console.log(
+        `${match.appmap}.appmap.json:${
+          match.eventIds[match.eventIds.length - 1]
+        } (score=${match.score})`
+      );
+    });
+  console.log();
+  console.log(`Suppressed printing of ${duplicateCount} duplicates`);
+};

packages/cli/src/lib/isNumeric.ts

@@ -0,0 +1,3 @@
+export default function isNumeric(n: string): boolean {
+  return !isNaN(parseFloat(n)) && isFinite(parseFloat(n));
+}

packages/cli/src/search/findStack.ts

@@ -0,0 +1,139 @@
+import { buildAppMap, CodeObject, Event } from '@appland/models';
+import { readFile } from 'fs/promises';
+import { inspect } from 'util';
+import { verbose } from '../utils';
+import LocationMap from './locationMap';
+import SearchResultHashV2 from './searchResultHashV2';
+
+export type FindStackMatch = {
+  appmap: string;
+  eventIds: number[];
+  score: number;
+  hash_v2: string;
+};
+
+export default class FindStack {
+  constructor(public appMapName: string, public stackLines: string[]) {}
+
+  async match(): Promise<FindStackMatch[]> {
+    const appmapFile = [this.appMapName, 'appmap.json'].join('.');
+
+    let appmapData: string;
+    try {
+      appmapData = JSON.parse(await readFile(appmapFile, 'utf-8'));
+    } catch (e) {
+      console.log((e as any).code);
+      console.warn(`Error loading ${appmapFile}: ${e}`);
+      return [];
+    }
+
+    const appmap = buildAppMap(appmapData).build();
+    const locationMap = new LocationMap(appmap.classMap);
+
+    const locationStack = [...this.stackLines];
+    if (verbose())
+      console.log(`Searching for stack: ${inspect(locationStack)}`);
+    const result: FindStackMatch[] = [];
+    let score: number[] = [];
+    let stack: Event[] = [];
+
+    const enter = (event: Event): boolean | undefined => {
+      let matchIndex: number | undefined;
+      if (event.path && event.lineno) {
+        if (verbose())
+          console.log(
+            `${stack.map((_) => '  ').join('')}${event.path}:${event.lineno}`
+          );
+        for (
+          let i = 0;
+          matchIndex === undefined && i < locationStack.length;
+          i++
+        ) {
+          const [stackLinePath, stackLineLineno] = locationStack[i].split(
+            ':',
+            2
+          );
+
+          if (stackLinePath === event.path)
+            locationMap.functionContainsLine(
+              stackLinePath,
+              event.lineno,
+              parseFloat(stackLineLineno)
+            );
+
+          if (
+            stackLinePath === event.path &&
+            locationMap.functionContainsLine(
+              stackLinePath,
+              event.lineno,
+              parseFloat(stackLineLineno)
+            )
+          ) {
+            matchIndex = i;
+          }
+        }
+      }
+
+      stack.push(event);
+      if (matchIndex !== undefined) {
+        if (verbose())
+          console.log(
+            `${stack.map((_) => '  ').join('')}Matched ${
+              locationStack[matchIndex]
+            } at event ${event.id} (${event.codeObject.fqid})`
+          );
+        locationStack.splice(0, matchIndex + 1);
+        if (verbose())
+          console.log(`Now matching stack: ${inspect(locationStack)}`);
+
+        score.push(1);
+        if (locationStack.length === 0) {
+          return true;
+        }
+      } else {
+        score.push(0);
+      }
+    };
+
+    const leave = () => {
+      stack.pop();
+      score.pop();
+    };
+
+    const eventsEmitted = new Set<number>();
+    for (let i = 0; i < appmap.events.length; ) {
+      const event = appmap.events[i];
+      if (event.isCall()) {
+        const isFullMatch = enter(event);
+        const isLeaf = event.children.length === 0;
+        if (isFullMatch || isLeaf) {
+          const total = score.reduce((sum, n) => (n ? sum + n : sum));
+          if (total > 0) {
+            const matchStack = stack.filter((_, index) => score[index]);
+            if (!eventsEmitted.has(matchStack[matchStack.length - 1].id)) {
+              eventsEmitted.add(matchStack[matchStack.length - 1].id);
+              const hash = new SearchResultHashV2(matchStack);
+              if (verbose()) console.log(`Match hash: ${hash.canonicalString}`);
+              result.push({
+                appmap: this.appMapName,
+                eventIds: matchStack.map((e) => e.id),
+                hash_v2: hash.digest(),
+                score: total,
+              });
+            }
+          }
+        }
+        if (isFullMatch) {
+          for (++i; appmap.events[i] !== event.returnEvent; ++i) {}
+        } else {
+          ++i;
+        }
+      } else {
+        ++i;
+        leave();
+      }
+    }
+
+    return result;
+  }
+}

packages/cli/src/search/locationMap.ts

@@ -0,0 +1,76 @@
+import { AppMap, ClassMap, CodeObject } from '@appland/models';
+import isNumeric from '../lib/isNumeric';
+
+export const Threshold = 20;
+
+export default class LocationMap {
+  lineNumbers: Map<string, number[]>;
+
+  constructor(public classMap: ClassMap) {
+    const lineNumbers: Map<string, number[]> = new Map();
+    classMap.visit((co: CodeObject) => {
+      if (!co.location) return;
+
+      const [path, lineno] = co.location.split(':', 2);
+      if (!path || !isNumeric(lineno)) return;
+
+      if (!lineNumbers.get(path)) lineNumbers.set(path, []);
+
+      lineNumbers.get(path)!.push(parseFloat(lineno));
+    });
+
+    this.lineNumbers = new Map<string, number[]>();
+    for (const entry of lineNumbers) {
+      this.lineNumbers.set(entry[0], [...entry[1].sort((a, b) => a - b)]);
+    }
+  }
+
+  /**
+   * Tests whether a test line number is contained within the function that starts at a given
+   * line number.
+   *
+   * If the test line number is greater than the given line number and less than the
+   * next known function line, returns true.
+   *
+   * If the test line number is greater than the given line number and greater than the
+   * next known function line, returns false.
+   *
+   * If the test line number is greater than the given line number, and there is no greater
+   * known function line, returns true if the test line number is within a threshold of
+   * the given line number.
+   *
+   * @param path file path containing the function to test
+   * @param lineNumber start line of the function
+   * @param testLineNumber line number to test
+   * @param threshold used if the testLineNumber is greater than the lineNumber, and there is
+   * no known larger function line number in the code file (path).
+   */
+  functionContainsLine(
+    path: string,
+    lineNumber: number,
+    testLineNumber: number,
+    threshold = Threshold
+  ): boolean {
+    if (testLineNumber < lineNumber) return false;
+
+    const lineNumbers = this.lineNumbers.get(path);
+    if (!lineNumbers) return false;
+
+    const lastLineNumber = lineNumbers[lineNumbers.length - 1];
+
+    let containingFunctionLine: number | undefined;
+    for (let i = 0; i < lineNumbers.length; i++) {
+      const line = lineNumbers[i];
+      if (line <= testLineNumber) containingFunctionLine = line;
+      else break;
+    }
+
+    if (lineNumber === containingFunctionLine) {
+      if (containingFunctionLine === lastLineNumber)
+        return testLineNumber - lastLineNumber < threshold;
+      else return true;
+    }
+
+    return false;
+  }
+}