From f5968a606ee11112f4a359dc68e37c51befb2cb7 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:11:26 +0100 Subject: [PATCH] Update: refactor caching * Removed duplicate `readFile` * More efficient checking, restoring and updating --- packages/core/src/cache.ts | 165 ++++++++++++++++++------------------- packages/core/src/index.ts | 92 +++++++++------------ 2 files changed, 122 insertions(+), 135 deletions(-) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 66e03d2..71f655c 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,6 +1,5 @@ import crypto, { BinaryLike } from 'node:crypto' -import { existsSync } from 'node:fs' -import { copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { isAbsolute, resolve } from 'node:path' import { normalizePath } from 'vite' @@ -11,7 +10,7 @@ import { smartEnsureDirs, } from './utils' -import type { CacheValue, ResolvedConfigOptions, StackItem } from './typings' +import type { CacheValue, ResolvedConfigOptions } from './typings' let cacheEnabled = false let cacheDir = '' @@ -87,36 +86,61 @@ async function initCacheMaps() { fileCacheMap = new Map(entryMap) } -async function getAndUpdateCacheContent( - filePath: string, - stats?: Omit, -): Promise<{ +async function checkAndUpdate({ + fileName, + directory, + stats, + buffer, + restoreTo, +}: { + fileName?: string + directory?: string + stats?: Omit + buffer?: Buffer + restoreTo?: string | false +}): Promise<{ changed?: boolean value?: CacheValue error?: Error }> { - let hash = '' - try { - hash = md5(await readFile(filePath)) - } catch (error) { - return { - error: error as Error, + if (cacheEnabled) { + const filePath = (directory ?? cacheDir) + fileName + + if (!buffer) { + try { + buffer = await readFile(filePath) + } catch (error) { + return { + error: error as Error, + } + } } - } - const cacheValue = fileCacheMap.get(filePath) - if (cacheValue && cacheValue.hash === hash) { - return { - changed: false, - value: cacheValue, + const hash = md5(buffer) + const cacheValue = fileCacheMap.get(filePath) + if (cacheValue && cacheValue.hash === hash) { + if (restoreTo) { + try { + await writeFile(restoreTo + fileName, buffer) + } catch (error) { + return { + error: error as Error, + } + } + } + + return { + changed: false, + value: cacheValue, + } } - } - entryMap.set(filePath, { - hash, - oldSize: stats?.oldSize ?? 0, - newSize: stats?.newSize ?? 0, - }) + entryMap.set(filePath, { + hash, + oldSize: stats?.oldSize ?? 1, + newSize: stats?.newSize ?? 1, + }) + } return { changed: true, @@ -144,72 +168,47 @@ export const FileCache = { smartEnsureDirs(filePaths.map(file => cacheDir + file)) }, - checkAndCopy: async ( - baseDir: string, - filePathFrom: string, - fileToStack: StackItem[] = [], - ): Promise<[boolean, (string | CacheValue)[]]> => { + checkAndUpdate: checkAndUpdate, + + update: async ({ + fileName, + directory, + stats, + buffer, + }: { + fileName?: string + directory?: string + stats?: Omit + buffer: Buffer + }) => { if (!cacheEnabled) { - return [false, []] + return false } - const inputCacheStatus = await getAndUpdateCacheContent( - baseDir + filePathFrom, - ) - - // Check if input file has changed or there was an error - if (inputCacheStatus?.error || inputCacheStatus?.changed) { - return [false, []] + if (!buffer) { + return { + error: new Error('Missing content for cache file'), + } } - // Check if output files are in cache and use them if they haven't changed - const outputFilesExist = await Promise.allSettled( - fileToStack.map(async item => { - const outputCacheStatus = await getAndUpdateCacheContent( - cacheDir + item.toPath, - ) - - if (outputCacheStatus?.error) { - return `Cache error [${outputCacheStatus.error.message}]` - } - - if (outputCacheStatus?.changed) { - return 'File changed' - } - - try { - await copyFile(cacheDir + item.toPath, baseDir + item.toPath) - } catch (error) { - return `Could not copy cached file [${(error as Error).message}]` - } - - if (!existsSync(baseDir + item.toPath)) { - return 'Could not use cached file' - } + const filePath = (directory ?? cacheDir) + fileName - return outputCacheStatus.value as CacheValue - }), - ) - - return [ - true, - outputFilesExist.map(p => - p.status === 'fulfilled' ? p.value : p.reason, - ), - ] - }, - - update: async ( - baseDir: string, - filePathTo: string, - stats: Omit, - ) => { - if (!cacheEnabled) { - return + try { + await writeFile(filePath, buffer) + } catch (error) { + return { + error: new Error( + `Could not write cache file [${(error as Error).message}]`, + ), + } } - await copyFile(baseDir + filePathTo, cacheDir + filePathTo) - await getAndUpdateCacheContent(cacheDir + filePathTo, stats) + return await checkAndUpdate({ + fileName, + directory, + buffer, + stats, + }) }, reconcile: async () => { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f1d1122..6f573e2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -252,50 +252,6 @@ export async function processFile({ }) } - let cacheStack: (false | Promise)[] = [] - - // TODO: Rewrite cache check and split this: - // - get hash from cache (if present) - // - after readFile below, make current hash and compare it - // - check output cache in process pipeline as well? - // - // Reason: Since FileCache.checkAndCopy needs to read the file for checking, - // we might as well do it only once as part of the process instead of twice. - const [inputCacheStatus, outputCacheStatus] = await FileCache.checkAndCopy( - baseDir, - filePathFrom, - fileToStack, - ) - - if (inputCacheStatus) { - let hasFullValidCache = true - - cacheStack = outputCacheStatus.map((status, i) => { - if (isString(status)) { - hasFullValidCache = false - return false - } - - return Promise.resolve( - formatProcessedFile({ - oldPath: filePathFrom, - newPath: fileToStack[i].toPath, - oldSize: status.oldSize, - newSize: status.newSize, - duration: 0, - fromCache: true, - precisions, - bytesDivider, - sizeUnit, - }), - ) - }) - - if (hasFullValidCache) { - return Promise.allSettled(cacheStack as Promise[]) - } - } - let oldBuffer: Buffer let oldSize = 0 @@ -320,16 +276,44 @@ export async function processFile({ }) } + const inputFileCacheStatus = await FileCache.checkAndUpdate({ + fileName: filePathFrom, + directory: baseDir, + buffer: oldBuffer, + restoreTo: false, + }) + const skipCache = Boolean( + inputFileCacheStatus?.error || inputFileCacheStatus?.changed, + ) + const start = performance.now() return Promise.allSettled( - fileToStack.map(async (item, i) => { - if (cacheStack[i]) { - return cacheStack[i] as Promise - } - + fileToStack.map(async item => { const filePathTo = item.toPath + if (!skipCache) { + const outputFileCacheStatus = await FileCache.checkAndUpdate({ + fileName: filePathTo, + restoreTo: baseDir, + }) + if (!outputFileCacheStatus?.error && !outputFileCacheStatus?.changed) { + return Promise.resolve( + formatProcessedFile({ + oldPath: filePathFrom, + newPath: filePathTo, + oldSize: outputFileCacheStatus?.value?.oldSize ?? 1, + newSize: outputFileCacheStatus?.value?.newSize ?? 1, + duration: 0, + fromCache: true, + precisions, + bytesDivider, + sizeUnit, + }), + ) + } + } + let newBuffer: Buffer let newSize = 0 @@ -368,9 +352,13 @@ export async function processFile({ } } - await FileCache.update(baseDir, filePathTo, { - oldSize, - newSize, + await FileCache.update({ + fileName: filePathTo, + buffer: newBuffer, + stats: { + oldSize, + newSize, + }, }) return Promise.resolve(