Merge pull request #1988 from cardstack/cs-7684-create-an-http-endpoi…

…nt-to-get-mtimes-for-all-files-in-a

Refactor indexing to get mtimes for entire realm in a single HTTP request

packages/host/app/lib/current-run.ts

@@ -228,7 +228,7 @@ export class CurrentRun {
 
   private async discoverInvalidations(
     url: URL,
-    mtimes: LastModifiedTimes,
+    indexMtimes: LastModifiedTimes,
   ): Promise<void> {
     log.debug(`discovering invalidations in dir ${url.href}`);
     let ignorePatterns = await this.#reader.readFile(
@@ -239,35 +239,23 @@ export class CurrentRun {
       this.#ignoreData[url.href] = ignorePatterns.content;
     }
 
-    let entries = await this.#reader.directoryListing(url);
-
-    for (let { url, kind, lastModified } of entries) {
-      let innerURL = new URL(url);
-      if (isIgnored(this.#realmURL, this.ignoreMap, innerURL)) {
+    let filesystemMtimes = await this.#reader.mtimes();
+    for (let [url, lastModified] of Object.entries(filesystemMtimes)) {
+      if (!url.endsWith('.json') && !hasExecutableExtension(url)) {
+        // Only allow json and executable files to be invalidated so that we
+        // don't end up with invalidated files that weren't meant to be indexed
+        // (images, etc)
         continue;
       }
-
-      if (kind === 'directory') {
-        await this.discoverInvalidations(innerURL, mtimes);
-      } else {
-        if (!url.endsWith('.json') && !hasExecutableExtension(url)) {
-          // Only allow json and executable files to be invalidated so that we don't end up with invalidated files that weren't meant to be indexed (images, etc)
-          continue;
-        }
-
-        let indexEntry = mtimes.get(innerURL.href);
-        if (
-          !indexEntry ||
-          indexEntry.type === 'error' ||
-          indexEntry.lastModified == null
-        ) {
-          await this.batch.invalidate(innerURL);
-          continue;
-        }
-
-        if (lastModified !== indexEntry.lastModified) {
-          await this.batch.invalidate(innerURL);
-        }
+      let indexEntry = indexMtimes.get(url);
+
+      if (
+        !indexEntry ||
+        indexEntry.type === 'error' ||
+        indexEntry.lastModified == null ||
+        lastModified !== indexEntry.lastModified
+      ) {
+        await this.batch.invalidate(new URL(url));
       }
     }
   }

packages/host/tests/integration/realm-indexing-and-querying-test.gts

@@ -3401,6 +3401,12 @@ module(`Integration | realm indexing and querying`, function (hooks) {
                 { id: mangoID },
                 {
                   id: vanGoghID,
+                  firstName: 'Van Gogh',
+                  friends: [
+                    {
+                      id: hassanID,
+                    },
+                  ],
                 },
               ],
             },

packages/realm-server/tests/helpers/index.ts

@@ -1,4 +1,4 @@
-import { writeFileSync, writeJSONSync } from 'fs-extra';
+import { writeFileSync, writeJSONSync, readdirSync, statSync } from 'fs-extra';
 import { NodeAdapter } from '../../node-realm';
 import { resolve, join } from 'path';
 import {
@@ -14,14 +14,16 @@ import {
   maybeHandleScopedCSSRequest,
   insertPermissions,
   IndexWriter,
-  type MatrixConfig,
-  type QueuePublisher,
-  type QueueRunner,
-  type IndexRunner,
   asExpressions,
   query,
   insert,
   param,
+  unixTime,
+  RealmPaths,
+  type MatrixConfig,
+  type QueuePublisher,
+  type QueueRunner,
+  type IndexRunner,
 } from '@cardstack/runtime-common';
 import { dirSync } from 'tmp';
 import { getLocalConfig as getSynapseConfig } from '../../synapse';
@@ -401,6 +403,7 @@ export async function runTestRealmServer({
   let testRealmHttpServer = testRealmServer.listen(parseInt(realmURL.port));
   await testRealmServer.start();
   return {
+    testRealmDir,
     testRealm,
     testRealmServer,
     testRealmHttpServer,
@@ -494,3 +497,28 @@ export async function fetchSubscriptionsByUserId(
     stripeSubscriptionId: result.stripe_subscription_id,
   }));
 }
+
+export function mtimes(
+  path: string,
+  realmURL: URL,
+): { [path: string]: number } {
+  const mtimes: { [path: string]: number } = {};
+  let paths = new RealmPaths(realmURL);
+
+  function traverseDir(currentPath: string) {
+    const entries = readdirSync(currentPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = join(currentPath, entry.name);
+      if (entry.isDirectory()) {
+        traverseDir(fullPath);
+      } else if (entry.isFile()) {
+        const stats = statSync(fullPath);
+        mtimes[paths.fileURL(fullPath.substring(path.length)).href] = unixTime(
+          stats.mtime.getTime(),
+        );
+      }
+    }
+  }
+  traverseDir(path);
+  return mtimes;
+}

packages/realm-server/tests/realm-server-test.ts

@@ -54,6 +54,7 @@ import {
   testRealmInfo,
   insertUser,
   insertPlan,
+  mtimes,
   fetchSubscriptionsByUserId,
   cleanWhiteSpace,
 } from './helpers';
@@ -152,6 +153,7 @@ module('Realm Server', function (hooks) {
   }
 
   let testRealm: Realm;
+  let testRealmPath: string;
   let testRealmHttpServer: Server;
   let request: SuperTest<Test>;
   let dir: DirResult;
@@ -173,7 +175,11 @@ module('Realm Server', function (hooks) {
           copySync(join(__dirname, 'cards'), testRealmDir);
         }
         let virtualNetwork = createVirtualNetwork();
-        ({ testRealm, testRealmHttpServer } = await runTestRealmServer({
+        ({
+          testRealm,
+          testRealmHttpServer,
+          testRealmDir: testRealmPath,
+        } = await runTestRealmServer({
           virtualNetwork,
           testRealmDir,
           realmsRootPath: join(dir.name, 'realm_server_1'),
@@ -210,6 +216,50 @@ module('Realm Server', function (hooks) {
     resetCatalogRealms();
   });
 
+  module('mtimes requests', function (hooks) {
+    setupPermissionedRealm(hooks, {
+      mary: ['read'],
+    });
+
+    test('non read permission GET /_mtimes', async function (assert) {
+      let response = await request
+        .get('/_mtimes')
+        .set('Accept', 'application/vnd.api+json')
+        .set('Authorization', `Bearer ${createJWT(testRealm, 'not-mary')}`);
+
+      assert.strictEqual(response.status, 403, 'HTTP 403 status');
+    });
+
+    test('read permission GET /_mtimes', async function (assert) {
+      let expectedMtimes = mtimes(testRealmPath, testRealmURL);
+      delete expectedMtimes[`${testRealmURL}.realm.json`];
+
+      let response = await request
+        .get('/_mtimes')
+        .set('Accept', 'application/vnd.api+json')
+        .set(
+          'Authorization',
+          `Bearer ${createJWT(testRealm, 'mary', ['read'])}`,
+        );
+
+      assert.strictEqual(response.status, 200, 'HTTP 200 status');
+      let json = response.body;
+      assert.deepEqual(
+        json,
+        {
+          data: {
+            type: 'mtimes',
+            id: testRealmHref,
+            attributes: {
+              mtimes: expectedMtimes,
+            },
+          },
+        },
+        'mtimes response is correct',
+      );
+    });
+  });
+
   module('permissions requests', function (hooks) {
     setupPermissionedRealm(hooks, {
       mary: ['read', 'write', 'realm-owner'],

packages/runtime-common/realm.ts

@@ -349,6 +349,7 @@ export class Realm {
         this.patchCard.bind(this),
       )
       .get('/_info', SupportedMimeType.RealmInfo, this.realmInfo.bind(this))
+      .get('/_mtimes', SupportedMimeType.Mtimes, this.realmMtimes.bind(this))
       .get('/_search', SupportedMimeType.CardJson, this.search.bind(this))
       .get(
         '/_search-prerendered',
@@ -1654,6 +1655,57 @@ export class Realm {
     });
   }
 
+  private async realmMtimes(
+    _request: Request,
+    requestContext: RequestContext,
+  ): Promise<Response> {
+    let mtimes: { [path: string]: number } = {};
+    let traverse = async (currentPath = '') => {
+      const entries = this.#adapter.readdir(currentPath);
+
+      for await (const entry of entries) {
+        let innerPath = join(currentPath, entry.name);
+        let innerURL =
+          entry.kind === 'directory'
+            ? this.paths.directoryURL(innerPath)
+            : this.paths.fileURL(innerPath);
+        if (await this.isIgnored(innerURL)) {
+          continue;
+        }
+        if (entry.kind === 'directory') {
+          await traverse(innerPath);
+        } else if (entry.kind === 'file') {
+          let mtime = await this.#adapter.lastModified(innerPath);
+          if (mtime != null) {
+            mtimes[innerURL.href] = mtime;
+          }
+        }
+      }
+    };
+
+    await traverse();
+
+    return createResponse({
+      body: JSON.stringify(
+        {
+          data: {
+            id: this.url,
+            type: 'mtimes',
+            attributes: {
+              mtimes,
+            },
+          },
+        },
+        null,
+        2,
+      ),
+      init: {
+        headers: { 'content-type': SupportedMimeType.Mtimes },
+      },
+      requestContext,
+    });
+  }
+
   private async getRealmPermissions(
     _request: Request,
     requestContext: RequestContext,

packages/runtime-common/router.ts

@@ -21,6 +21,7 @@ export enum SupportedMimeType {
   CardSource = 'application/vnd.card+source',
   DirectoryListing = 'application/vnd.api+json',
   RealmInfo = 'application/vnd.api+json',
+  Mtimes = 'application/vnd.api+json',
   Permissions = 'application/vnd.api+json',
   Session = 'application/json',
   EventStream = 'text/event-stream',

packages/runtime-common/worker.ts

@@ -15,7 +15,6 @@ import {
   type QueueRunner,
   type TextFileRef,
   type VirtualNetwork,
-  type Relationship,
   type ResponseWithNodeStream,
 } from '.';
 import { MatrixClient } from './matrix-client';
@@ -36,11 +35,7 @@ export interface IndexResults {
 
 export interface Reader {
   readFile: (url: URL) => Promise<TextFileRef | undefined>;
-  directoryListing: (
-    url: URL,
-  ) => Promise<
-    { kind: 'directory' | 'file'; url: string; lastModified: number | null }[]
-  >;
+  mtimes: () => Promise<{ [url: string]: number }>;
 }
 
 export type RunnerRegistration = (
@@ -342,29 +337,20 @@ export function getReader(
       };
     },
 
-    directoryListing: async (url: URL) => {
-      let response = await _fetch(url, {
+    mtimes: async () => {
+      let response = await _fetch(`${realmURL.href}_mtimes`, {
         headers: {
-          Accept: SupportedMimeType.DirectoryListing,
+          Accept: SupportedMimeType.Mtimes,
         },
       });
       let {
-        data: { relationships: _relationships },
-      } = await response.json();
-      let relationships = _relationships as Record<string, Relationship>;
-      return Object.values(relationships).map((entry) =>
-        entry.meta!.kind === 'file'
-          ? {
-              url: entry.links.related!,
-              kind: 'file',
-              lastModified: (entry.meta?.lastModified ?? null) as number | null,
-            }
-          : {
-              url: entry.links.related!,
-              kind: 'directory',
-              lastModified: null,
-            },
-      );
+        data: {
+          attributes: { mtimes },
+        },
+      } = (await response.json()) as {
+        data: { attributes: { mtimes: { [url: string]: number } } };
+      };
+      return mtimes;
     },
   };
 }