[ENG-9500][WIP] Caching for custom builds

[ENG-9500] Parse template strings with function calls

[ENG-9500] Implement input functions (hashFiles)

[ENG-9500] Implement new interface for CacheManager

`CacheManager.saveCache` and `CacheManager.restoreCache` now separete context from `Cache`, making them more flexible when calling cache functions.
Since turtle-v2 implementation of CacheManager (`GCSCacheManager`) expected context to have defined `job` property just for the purpose of extracting certain paths in project, `BuildContext` was replaced with `CacheableContext`, which which expects only relevant properties to be defined. To satisfy this new interface, additional fields (`projectRootDirectory` and `buildDirectory`) were added to various contexts.

[ENG-9500] Small refactors

[ENG-9500] Implement cache function

[ENG-9500] Cache step for custom workflow

packages/build-tools/src/builders/android.ts

@@ -59,7 +59,10 @@ async function buildAsync(ctx: BuildContext<Android.Job>): Promise<void> {
   });
 
   await ctx.runBuildPhase(BuildPhase.RESTORE_CACHE, async () => {
-    await ctx.cacheManager?.restoreCache(ctx);
+    await ctx.cacheManager?.restoreCache(
+      { global: ctx, logger: ctx.logger, workingdir: ctx.workingdir },
+      ctx.job.cache
+    );
   });
 
   await ctx.runBuildPhase(BuildPhase.POST_INSTALL_HOOK, async () => {
@@ -95,7 +98,10 @@ async function buildAsync(ctx: BuildContext<Android.Job>): Promise<void> {
   });
 
   await ctx.runBuildPhase(BuildPhase.SAVE_CACHE, async () => {
-    await ctx.cacheManager?.saveCache(ctx);
+    await ctx.cacheManager?.saveCache(
+      { global: ctx, logger: ctx.logger, workingdir: ctx.workingdir },
+      ctx.job.cache
+    );
   });
 
   await ctx.runBuildPhase(BuildPhase.UPLOAD_APPLICATION_ARCHIVE, async () => {

packages/build-tools/src/builders/ios.ts

@@ -63,7 +63,10 @@ async function buildAsync(ctx: BuildContext<Ios.Job>): Promise<void> {
     });
 
     await ctx.runBuildPhase(BuildPhase.RESTORE_CACHE, async () => {
-      await ctx.cacheManager?.restoreCache(ctx);
+      await ctx.cacheManager?.restoreCache(
+        { global: ctx, logger: ctx.logger, workingdir: ctx.workingdir },
+        ctx.job.cache
+      );
     });
 
     await ctx.runBuildPhase(BuildPhase.INSTALL_PODS, async () => {
@@ -106,7 +109,10 @@ async function buildAsync(ctx: BuildContext<Ios.Job>): Promise<void> {
   });
 
   await ctx.runBuildPhase(BuildPhase.SAVE_CACHE, async () => {
-    await ctx.cacheManager?.saveCache(ctx);
+    await ctx.cacheManager?.saveCache(
+      { global: ctx, logger: ctx.logger, workingdir: ctx.workingdir },
+      ctx.job.cache
+    );
   });
 
   await ctx.runBuildPhase(BuildPhase.UPLOAD_APPLICATION_ARCHIVE, async () => {

packages/build-tools/src/context.ts

@@ -16,12 +16,15 @@ import { ExpoConfig } from '@expo/config';
 import { bunyan } from '@expo/logger';
 import { SpawnPromise, SpawnOptions, SpawnResult } from '@expo/turtle-spawn';
 import { BuildTrigger } from '@expo/eas-build-job/dist/common';
+import { CacheManager } from '@expo/steps';
 
 import { PackageManager, resolvePackageManager } from './utils/packageManager';
 import { resolveBuildPhaseErrorAsync } from './buildErrors/detectError';
 import { readAppConfig } from './utils/appConfig';
 import { createTemporaryEnvironmentSecretFile } from './utils/environmentSecrets';
 
+export { CacheManager } from '@expo/steps';
+
 export enum ArtifactType {
   APPLICATION_ARCHIVE = 'APPLICATION_ARCHIVE',
   BUILD_ARTIFACTS = 'BUILD_ARTIFACTS',
@@ -33,11 +36,6 @@ export enum ArtifactType {
 
 export type Artifacts = Partial<Record<ArtifactType, string>>;
 
-export interface CacheManager {
-  saveCache(ctx: BuildContext<Job>): Promise<void>;
-  restoreCache(ctx: BuildContext<Job>): Promise<void>;
-}
-
 export interface LogBuffer {
   getLogs(): string[];
   getPhaseLogs(buildPhase: string): string[];
@@ -132,6 +130,10 @@ export class BuildContext<TJob extends Job> {
       : this.buildExecutablesDirectory;
   }
 
+  get projectRootDirectory(): string | undefined {
+    return this.job.projectRootDirectory;
+  }
+
   public get job(): TJob {
     return this._job;
   }

packages/build-tools/src/customBuildContext.ts

@@ -2,7 +2,7 @@ import path from 'path';
 
 import { BuildPhase, Env, Job, Metadata, Platform } from '@expo/eas-build-job';
 import { bunyan } from '@expo/logger';
-import { ExternalBuildContextProvider, BuildRuntimePlatform } from '@expo/steps';
+import { ExternalBuildContextProvider, BuildRuntimePlatform, CacheManager } from '@expo/steps';
 
 import { ArtifactType, BuildContext } from './context';
 
@@ -40,6 +40,9 @@ export class CustomBuildContext implements ExternalBuildContextProvider {
   public readonly runtimeApi: BuilderRuntimeApi;
   public readonly job: Job;
   public readonly metadata?: Metadata;
+  public readonly cacheManager?: CacheManager;
+  public readonly buildDirectory: string;
+  public readonly projectRootDirectory: string;
 
   private _env: Env;
 
@@ -56,6 +59,9 @@ export class CustomBuildContext implements ExternalBuildContextProvider {
     this.runtimeApi = {
       uploadArtifacts: (...args) => buildCtx['uploadArtifacts'](...args),
     };
+    this.cacheManager = buildCtx.cacheManager;
+    this.buildDirectory = buildCtx.buildDirectory;
+    this.projectRootDirectory = buildCtx.projectRootDirectory ?? '.';
   }
 
   public get runtimePlatform(): BuildRuntimePlatform {

packages/build-tools/src/index.ts

@@ -2,17 +2,17 @@ import * as Builders from './builders';
 
 export { Builders };
 
+export { PackageManager } from './utils/packageManager';
+
 export {
   Artifacts,
   ArtifactType,
   BuildContext,
-  CacheManager,
   LogBuffer,
   SkipNativeBuildError,
+  CacheManager,
 } from './context';
 
-export { PackageManager } from './utils/packageManager';
-
 export { findAndUploadXcodeBuildLogsAsync } from './ios/xcodeBuildLogs';
 
 export { Hook, runHookIfPresent } from './utils/hooks';

packages/build-tools/src/steps/easFunctions.ts

@@ -8,6 +8,7 @@ import { createSetUpNpmrcBuildFunction } from './functions/useNpmToken';
 import { createInstallNodeModulesBuildFunction } from './functions/installNodeModules';
 import { createPrebuildBuildFunction } from './functions/prebuild';
 import { createFindAndUploadBuildArtifactsBuildFunction } from './functions/findAndUploadBuildArtifacts';
+import { createSaveCacheBuildFunction, createRestoreCacheBuildFunction } from './functions/cache';
 import { configureEASUpdateIfInstalledFunction } from './functions/configureEASUpdateIfInstalled';
 import { injectAndroidCredentialsFunction } from './functions/injectAndroidCredentials';
 import { configureAndroidVersionFunction } from './functions/configureAndroidVersion';
@@ -26,6 +27,8 @@ export function getEasFunctions(ctx: CustomBuildContext): BuildFunction[] {
     createInstallNodeModulesBuildFunction(),
     createPrebuildBuildFunction(),
     createFindAndUploadBuildArtifactsBuildFunction(ctx),
+    createSaveCacheBuildFunction(),
+    createRestoreCacheBuildFunction(),
     configureEASUpdateIfInstalledFunction(),
     injectAndroidCredentialsFunction(),
     configureAndroidVersionFunction(),

packages/build-tools/src/steps/functions/__tests__/cache.test.ts

@@ -0,0 +1,136 @@
+import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
+
+import {
+  BuildRuntimePlatform,
+  BuildStepGlobalContext,
+  ExternalBuildContextProvider,
+  CacheManager,
+} from '@expo/steps';
+import { anything, capture, instance, mock, reset, verify, when } from 'ts-mockito';
+
+import { createLogger } from '../../../__mocks__/@expo/logger';
+import { createRestoreCacheBuildFunction, createSaveCacheBuildFunction } from '../cache';
+
+const cacheSaveBuildFunction = createSaveCacheBuildFunction();
+const cacheRestoreBuildFunction = createRestoreCacheBuildFunction();
+
+const providerMock = mock<ExternalBuildContextProvider>();
+const cacheManagerMock = mock<CacheManager>();
+
+const cacheManager = instance(cacheManagerMock);
+const initialCache = { downloadUrls: {} };
+
+const provider = instance(providerMock);
+
+let ctx: BuildStepGlobalContext;
+
+const existingKey =
+  'c7d8e33243968f8675ec0463ad89e11c1e754723695ab9b23dfb8f9ddd389a28-value-8b6e2366e2a2ff8b43556a1dcc5f1cf97ddcf4cdf3c8f9a6d54e0efe2e747922';
+
+describe('cache functions', () => {
+  let key: string;
+  let paths: string[];
+  beforeEach(async () => {
+    key = '${ hashFiles("./src/*")  }-value';
+    paths = ['path1', 'path2'];
+    reset(cacheManagerMock);
+    reset(providerMock);
+
+    const projectSourceDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'project-'));
+    when(providerMock.logger).thenReturn(createLogger());
+    when(providerMock.runtimePlatform).thenReturn(BuildRuntimePlatform.LINUX);
+    when(providerMock.staticContext()).thenReturn({ some: 'key', job: { cache: initialCache } });
+    when(providerMock.cacheManager).thenReturn(cacheManager);
+    when(providerMock.projectSourceDirectory).thenReturn(projectSourceDirectory);
+    when(providerMock.defaultWorkingDirectory).thenReturn(projectSourceDirectory);
+    when(providerMock.projectTargetDirectory).thenReturn(projectSourceDirectory);
+
+    ctx = new BuildStepGlobalContext(provider, false);
+
+    await fs.mkdir(path.join(projectSourceDirectory, 'src'));
+    await fs.writeFile(path.join(projectSourceDirectory, 'src', 'path1'), 'placeholder');
+    await fs.writeFile(path.join(projectSourceDirectory, 'src', 'path2'), 'placeholder');
+  });
+
+  describe('cacheRestoreBuildFunction', () => {
+    test('has correct identifiers', () => {
+      expect(cacheRestoreBuildFunction.id).toBe('restore-cache');
+      expect(cacheRestoreBuildFunction.namespace).toBe('eas');
+      expect(cacheRestoreBuildFunction.name).toBe('Restore Cache');
+    });
+
+    test('restores cache if it exists', async () => {
+      when(cacheManagerMock.restoreCache(anything(), anything()));
+      initialCache.downloadUrls = { [existingKey]: 'url' };
+
+      const buildStep = cacheRestoreBuildFunction.createBuildStepFromFunctionCall(ctx, {
+        callInputs: { key, paths },
+      });
+
+      when(providerMock.defaultWorkingDirectory).thenReturn('/tmp');
+
+      await buildStep.executeAsync();
+
+      verify(cacheManagerMock.restoreCache(anything(), anything())).once();
+
+      const [, cache] = capture(cacheManagerMock.restoreCache).first();
+      expect(cache.key).toMatch(/^\w+-value/);
+      expect(cache.paths).toStrictEqual(paths);
+    });
+
+    test("doesn't restore cache if it doesn't exist", async () => {
+      when(cacheManagerMock.restoreCache(anything(), anything()));
+      initialCache.downloadUrls = { invalidkey: 'url' };
+
+      const buildStep = cacheRestoreBuildFunction.createBuildStepFromFunctionCall(ctx, {
+        callInputs: { key, paths },
+      });
+
+      await buildStep.executeAsync();
+
+      verify(cacheManagerMock.restoreCache(anything(), anything())).never();
+    });
+  });
+
+  describe('cacheSaveBuildFunction', () => {
+    test('has correct identifiers', () => {
+      expect(cacheSaveBuildFunction.id).toBe('save-cache');
+      expect(cacheSaveBuildFunction.namespace).toBe('eas');
+      expect(cacheSaveBuildFunction.name).toBe('Save Cache');
+    });
+
+    test('saves cache if it does not exist', async () => {
+      when(cacheManagerMock.restoreCache(anything(), anything()));
+
+      initialCache.downloadUrls = {};
+
+      const buildStep = cacheSaveBuildFunction.createBuildStepFromFunctionCall(ctx, {
+        callInputs: { key, paths },
+      });
+
+      await buildStep.executeAsync();
+
+      verify(cacheManagerMock.saveCache(anything(), anything())).once();
+
+      const [, cache] = capture(cacheManagerMock.saveCache).first();
+      expect(cache.key).toMatch(/^\w+-value/);
+      expect(cache.paths).toStrictEqual(paths);
+    });
+
+    test("doesn't save cache if it exists", async () => {
+      when(cacheManagerMock.restoreCache(anything(), anything()));
+
+      initialCache.downloadUrls = { [existingKey]: 'url' };
+
+      const buildStep = cacheSaveBuildFunction.createBuildStepFromFunctionCall(ctx, {
+        callInputs: { key, paths },
+      });
+
+      await buildStep.executeAsync();
+
+      verify(cacheManagerMock.saveCache(anything(), anything())).never();
+    });
+  });
+});