Skip to content

Commit

Permalink
Merge pull request backstage#27215 from backstage/rugvip/add-cache
Browse files Browse the repository at this point in the history
cli: refactor success cache to be additive
  • Loading branch information
Rugvip authored Oct 19, 2024
2 parents 4e87994 + 1ff8ca3 commit 6cd1052
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 66 deletions.
5 changes: 5 additions & 0 deletions .changeset/loud-trainers-yawn.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@backstage/cli': patch
---

The `--successCache` option for the `repo test` and `repo lint` commands now use an additive store that keeps old entries around for a week before they are cleaned up automatically.
40 changes: 7 additions & 33 deletions packages/cli/src/commands/repo/lint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@

import chalk from 'chalk';
import { Command, OptionValues } from 'commander';
import fs from 'fs-extra';
import { createHash } from 'crypto';
import { relative as relativePath, resolve as resolvePath } from 'path';
import { relative as relativePath } from 'path';
import {
PackageGraph,
BackstagePackageJson,
Expand All @@ -27,6 +26,7 @@ import {
import { paths } from '../../lib/paths';
import { runWorkerQueueThreads } from '../../lib/parallel';
import { createScriptOptionsParser } from './optionsParser';
import { SuccessCache } from '../../lib/cache/SuccessCache';

function depCount(pkg: BackstagePackageJson) {
const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0;
Expand All @@ -36,39 +36,13 @@ function depCount(pkg: BackstagePackageJson) {
return deps + devDeps;
}

const CACHE_FILE_NAME = 'lint-cache.json';

type Cache = string[];

async function readCache(dir: string): Promise<Cache | undefined> {
try {
const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME));
if (!Array.isArray(data)) {
return undefined;
}
if (data.some(x => typeof x !== 'string')) {
return undefined;
}
return data as Cache;
} catch {
return undefined;
}
}

async function writeCache(dir: string, cache: Cache) {
await fs.mkdirp(dir);
await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 });
}

export async function command(opts: OptionValues, cmd: Command): Promise<void> {
let packages = await PackageGraph.listTargetPackages();

const cacheDir = resolvePath(
opts.successCacheDir ?? 'node_modules/.cache/backstage-cli',
);
const cache = new SuccessCache('lint', opts.successCacheDir);
const cacheContext = opts.successCache
? {
cache: await readCache(cacheDir),
entries: await cache.read(),
lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')),
}
: undefined;
Expand Down Expand Up @@ -136,7 +110,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
fix: Boolean(opts.fix),
format: opts.format as string | undefined,
shouldCache: Boolean(cacheContext),
successCache: cacheContext?.cache,
successCache: cacheContext?.entries,
rootDir: paths.targetRoot,
},
workerFactory: async ({
Expand Down Expand Up @@ -202,7 +176,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
hash.update('\0');
}
sha = await hash.digest('hex');
if (successCache?.includes(sha)) {
if (successCache?.has(sha)) {
console.log(`Skipped ${relativeDir} due to cache hit`);
return { relativeDir, sha, failed: false };
}
Expand Down Expand Up @@ -262,7 +236,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
}

if (cacheContext) {
await writeCache(cacheDir, outputSuccessCache);
await cache.write(outputSuccessCache);
}

if (failed) {
Expand Down
39 changes: 6 additions & 33 deletions packages/cli/src/commands/repo/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@

import os from 'os';
import crypto from 'node:crypto';
import fs from 'fs-extra';
import yargs from 'yargs';
import { resolve as resolvePath, relative as relativePath } from 'path';
import { relative as relativePath } from 'path';
import { Command, OptionValues } from 'commander';
import { Lockfile, PackageGraph } from '@backstage/cli-node';
import { paths } from '../../lib/paths';
import { runCheck, runPlain } from '../../lib/run';
import { isChildPath } from '@backstage/cli-common';
import { SuccessCache } from '../../lib/cache/SuccessCache';

type JestProject = {
displayName: string;
Expand All @@ -50,30 +50,6 @@ interface GlobalWithCache extends Global {
};
}

const CACHE_FILE_NAME = 'test-cache.json';

type Cache = string[];

async function readCache(dir: string): Promise<Cache | undefined> {
try {
const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME));
if (!Array.isArray(data)) {
return undefined;
}
if (data.some(x => typeof x !== 'string')) {
return undefined;
}
return data as Cache;
} catch {
return undefined;
}
}

function writeCache(dir: string, cache: Cache) {
fs.mkdirpSync(dir);
fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 });
}

/**
* Use git to get the HEAD tree hashes of each package in the project.
*/
Expand Down Expand Up @@ -272,10 +248,6 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
removeOptionArg(args, '--successCache', 1);
removeOptionArg(args, '--successCacheDir');

const cacheDir = resolvePath(
opts.successCacheDir ?? 'node_modules/.cache/backstage-cli',
);

// Parse the args to ensure that no file filters are provided, in which case we refuse to run
const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions)
.argv;
Expand All @@ -293,6 +265,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
);
}

const cache = new SuccessCache('test', opts.successCacheDir);
const graph = await getPackageGraph();

// Shared state for the bridge
Expand All @@ -305,7 +278,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
globalWithCache.__backstageCli_jestSuccessCache = {
// This is called by `config/jest.js` after the project configs have been gathered
async filterConfigs(projectConfigs, globalRootConfig) {
const cache = await readCache(cacheDir);
const cacheEntries = await cache.read();
const lockfile = await Lockfile.load(
paths.resolveTargetRoot('yarn.lock'),
);
Expand Down Expand Up @@ -350,7 +323,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {

projectHashes.set(packageName, sha);

if (cache?.includes(sha)) {
if (cacheEntries.has(sha)) {
if (!selectedProjects || selectedProjects.includes(packageName)) {
console.log(`Skipped ${packageName} due to cache hit`);
}
Expand Down Expand Up @@ -391,7 +364,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
}
}

await writeCache(cacheDir, outputSuccessCache);
await cache.write(outputSuccessCache);
},
};
}
Expand Down
93 changes: 93 additions & 0 deletions packages/cli/src/lib/cache/SuccessCache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import fs from 'fs-extra';
import { resolve as resolvePath } from 'node:path';

const DEFAULT_CACHE_BASE_PATH = 'node_modules/.cache/backstage-cli';

const CACHE_MAX_AGE_MS = 7 * 24 * 3600_000;

export class SuccessCache {
readonly #path: string;

constructor(name: string, basePath?: string) {
this.#path = resolvePath(basePath ?? DEFAULT_CACHE_BASE_PATH, name);
}

async read(): Promise<Set<string>> {
try {
const stat = await fs.stat(this.#path);
if (!stat.isDirectory()) {
await fs.rm(this.#path);
return new Set();
}
} catch (error) {
if (error.code === 'ENOENT') {
return new Set();
}
throw error;
}

const items = await fs.readdir(this.#path);

const returned = new Set<string>();
const removed = new Set<string>();

const now = Date.now();

for (const item of items) {
const split = item.split('_');
if (split.length !== 2) {
removed.add(item);
continue;
}
const createdAt = parseInt(split[0], 10);
if (Number.isNaN(createdAt) || now - createdAt > CACHE_MAX_AGE_MS) {
removed.add(item);
} else {
returned.add(split[1]);
}
}

for (const item of removed) {
await fs.unlink(resolvePath(this.#path, item));
}

return returned;
}

async write(newEntries: Iterable<string>): Promise<void> {
const now = Date.now();

await fs.ensureDir(this.#path);

const existingItems = await fs.readdir(this.#path);

const empty = Buffer.alloc(0);
for (const key of newEntries) {
// Remove any existing items with the key we're about to add
const trimmedItems = existingItems.filter(item =>
item.endsWith(`_${key}`),
);
for (const trimmedItem of trimmedItems) {
await fs.unlink(resolvePath(this.#path, trimmedItem));
}

await fs.writeFile(resolvePath(this.#path, `${now}_${key}`), empty);
}
}
}

0 comments on commit 6cd1052

Please sign in to comment.