Install state file (yarnpkg#998)

* Add virtual state in options * Change some stuff * Update doc * Tests * again * ok * Come on * Sneaky Typescript * Made generateVirtualState look like generateLockFile * Add tim * time * find timing * Fix * Generate the right virtual state file * Fix * Compare old vs new option * No battery left on laptop, think Im almost done * Trying to fix * Fixes the serialization / hydratation * Add timing * Persist all like crazy * Call restore on load * Restore should start working... * RestoreState does not work... * JSON for install state * Works! 🙏 * update test * Cleaning * Optimized serialization * Remove install state * doc * Clean * Enable restoreInstallState on all commands * Cleanup * More cleanup * Control checksums to ensure cache is up to date * Small additional optimization * clean dirty stuff * Slight changes * Adds versions * Remove test "echo" script from package.json It was nice for testing but a bit out of place in the end :-) Co-Authored-By: Kristoffer K. <[email protected]> * Fixes dlx * Fixes constraints Co-authored-by: Maël Nison <[email protected]> Co-authored-by: Kristoffer K. <[email protected]>
bertho-zero · Feb 27, 2020 · 92c3fae · 92c3fae
1 parent 63aba82
commit 92c3fae
Show file tree

Hide file tree

Showing 18 changed files with 235 additions and 101 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,9 @@ package.tgz
 # This is the Yarn build state; it's local to each clone
 /.yarn/build-state.yml
 
+# This is the Yarn install state cache, it can be rebuilt anytime
+/.yarn/install-state.gz
+
 # Those files are meant to be local-only
 /packages/*/bundles
 

diff --git a/.pnp.js b/.pnp.js
diff --git a/.yarn/versions/1e50b08a.yml b/.yarn/versions/1e50b08a.yml
@@ -0,0 +1,29 @@
+releases:
+  "@yarnpkg/cli": prerelease
+  "@yarnpkg/core": prerelease
+  "@yarnpkg/plugin-dlx": prerelease
+  "@yarnpkg/plugin-essentials": prerelease
+  "@yarnpkg/plugin-npm-cli": prerelease
+  "@yarnpkg/plugin-pack": prerelease
+  "@yarnpkg/plugin-patch": prerelease
+  "@yarnpkg/plugin-version": prerelease
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-node-modules"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
diff --git a/packages/gatsby/content/advanced/questions-and-answers.md b/packages/gatsby/content/advanced/questions-and-answers.md
@@ -49,16 +49,18 @@ Lockfiles should **always** be kept within the repository. Continuous integratio
 If you're using Zero-Installs:
 
 ```gitignore
-.yarn/unplugged
-.yarn/build-state.yml
+.yarn/*
+!.yarn/cache
+!.yarn/releases
+!.yarn/plugins
 ```
 
 If you're not using Zero-Installs:
 
 ```gitignore
-.yarn/cache
-.yarn/unplugged
-.yarn/build-state.yml
+.yarn/*
+!.yarn/releases
+!.yarn/plugins
 .pnp.*
 ```
 

diff --git a/packages/plugin-dlx/sources/commands/dlx.ts b/packages/plugin-dlx/sources/commands/dlx.ts
@@ -1,5 +1,5 @@
 import {BaseCommand, WorkspaceRequiredError}                   from '@yarnpkg/cli';
-import {Configuration, Project, ThrowReport}                   from '@yarnpkg/core';
+import {Configuration, Project}                                from '@yarnpkg/core';
 import {scriptUtils, structUtils}                              from '@yarnpkg/core';
 import {Filename, PortablePath, npath, ppath, toFilename, xfs} from '@yarnpkg/fslib';
 import {Command, Usage}                                        from 'clipanion';
@@ -64,10 +64,7 @@ export default class DlxCommand extends BaseCommand {
       if (workspace === null)
         throw new WorkspaceRequiredError(project.cwd, tmpDir);
 
-      await project.resolveEverything({
-        lockfileOnly: true,
-        report: new ThrowReport(),
-      });
+      await project.restoreInstallState();
 
       return await scriptUtils.executeWorkspaceAccessibleBinary(workspace, command, this.args, {
         cwd: this.context.cwd,

diff --git a/packages/plugin-essentials/sources/commands/bin.ts b/packages/plugin-essentials/sources/commands/bin.ts
@@ -1,7 +1,7 @@
-import {BaseCommand}                                       from '@yarnpkg/cli';
-import {Configuration, Project, ThrowReport, StreamReport} from '@yarnpkg/core';
-import {scriptUtils, structUtils}                          from '@yarnpkg/core';
-import {Command, Usage, UsageError}                        from 'clipanion';
+import {BaseCommand}                          from '@yarnpkg/cli';
+import {Configuration, Project, StreamReport} from '@yarnpkg/core';
+import {scriptUtils, structUtils}             from '@yarnpkg/core';
+import {Command, Usage, UsageError}           from 'clipanion';
 
 // eslint-disable-next-line arca/no-default-export
 export default class BinCommand extends BaseCommand {
@@ -37,10 +37,7 @@ export default class BinCommand extends BaseCommand {
     const configuration = await Configuration.find(this.context.cwd, this.context.plugins);
     const {project, locator} = await Project.find(configuration, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     if (this.name) {
       const binaries = await scriptUtils.getPackageAccessibleBinaries(locator, {project});

diff --git a/packages/plugin-essentials/sources/commands/run.ts b/packages/plugin-essentials/sources/commands/run.ts
@@ -1,9 +1,9 @@
-import {BaseCommand}                                    from '@yarnpkg/cli';
-import {Configuration, Project, Workspace, ThrowReport} from '@yarnpkg/core';
-import {scriptUtils, structUtils}                       from '@yarnpkg/core';
-import {Command, Usage, UsageError}                     from 'clipanion';
+import {BaseCommand}                       from '@yarnpkg/cli';
+import {Configuration, Project, Workspace} from '@yarnpkg/core';
+import {scriptUtils, structUtils}          from '@yarnpkg/core';
+import {Command, Usage, UsageError}        from 'clipanion';
 
-import {pluginCommands}                                 from '../pluginCommands';
+import {pluginCommands}                    from '../pluginCommands';
 
 // eslint-disable-next-line arca/no-default-export
 export default class RunCommand extends BaseCommand {
@@ -67,10 +67,7 @@ export default class RunCommand extends BaseCommand {
     const configuration = await Configuration.find(this.context.cwd, this.context.plugins);
     const {project, workspace, locator} = await Project.find(configuration, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     const effectiveLocator = this.topLevel
       ? project.topLevelWorkspace.anchoredLocator

diff --git a/packages/plugin-essentials/sources/commands/why.ts b/packages/plugin-essentials/sources/commands/why.ts
@@ -1,10 +1,10 @@
-import {BaseCommand, WorkspaceRequiredError}              from '@yarnpkg/cli';
-import {Configuration, LocatorHash, Package, ThrowReport} from '@yarnpkg/core';
-import {IdentHash, Project}                               from '@yarnpkg/core';
-import {miscUtils, structUtils}                           from '@yarnpkg/core';
-import {Command, Usage}                                   from 'clipanion';
-import {Writable}                                         from 'stream';
-import {asTree}                                           from 'treeify';
+import {BaseCommand, WorkspaceRequiredError} from '@yarnpkg/cli';
+import {Configuration, LocatorHash, Package} from '@yarnpkg/core';
+import {IdentHash, Project}                  from '@yarnpkg/core';
+import {miscUtils, structUtils}              from '@yarnpkg/core';
+import {Command, Usage}                      from 'clipanion';
+import {Writable}                            from 'stream';
+import {asTree}                              from 'treeify';
 
 type TreeNode = {[key: string]: TreeNode};
 
@@ -42,10 +42,7 @@ export default class WhyCommand extends BaseCommand {
     if (!workspace)
       throw new WorkspaceRequiredError(project.cwd, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     const identHash = structUtils.parseIdent(this.package).identHash;
 

diff --git a/packages/plugin-npm-cli/sources/commands/npm/publish.ts b/packages/plugin-npm-cli/sources/commands/npm/publish.ts
@@ -1,11 +1,11 @@
-import {BaseCommand, WorkspaceRequiredError}                                                    from '@yarnpkg/cli';
-import {Configuration, MessageName, Project, ReportError, StreamReport, Workspace, ThrowReport} from '@yarnpkg/core';
-import {miscUtils, structUtils}                                                                 from '@yarnpkg/core';
-import {npmConfigUtils, npmHttpUtils}                                                           from '@yarnpkg/plugin-npm';
-import {packUtils}                                                                              from '@yarnpkg/plugin-pack';
-import {Command, Usage, UsageError}                                                             from 'clipanion';
-import {createHash}                                                                             from 'crypto';
-import ssri                                                                                     from 'ssri';
+import {BaseCommand, WorkspaceRequiredError}                                       from '@yarnpkg/cli';
+import {Configuration, MessageName, Project, ReportError, StreamReport, Workspace} from '@yarnpkg/core';
+import {miscUtils, structUtils}                                                    from '@yarnpkg/core';
+import {npmConfigUtils, npmHttpUtils}                                              from '@yarnpkg/plugin-npm';
+import {packUtils}                                                                 from '@yarnpkg/plugin-pack';
+import {Command, Usage, UsageError}                                                from 'clipanion';
+import {createHash}                                                                from 'crypto';
+import ssri                                                                        from 'ssri';
 
 // eslint-disable-next-line arca/no-default-export
 export default class NpmPublishCommand extends BaseCommand {
@@ -47,10 +47,7 @@ export default class NpmPublishCommand extends BaseCommand {
     if (workspace.manifest.name === null || workspace.manifest.version === null)
       throw new UsageError(`Workspaces must have valid names and versions to be published on an external registry`);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     // We store it so that TS knows that it's non-null
     const ident = workspace.manifest.name;

diff --git a/packages/plugin-pack/sources/commands/pack.ts b/packages/plugin-pack/sources/commands/pack.ts
@@ -60,10 +60,7 @@ export default class PackCommand extends BaseCommand {
           report: new ThrowReport(),
         });
       } else {
-        await project.resolveEverything({
-          lockfileOnly: true,
-          report: new ThrowReport(),
-        });
+        await project.restoreInstallState();
       }
     }
 

diff --git a/packages/plugin-patch/sources/commands/patch.ts b/packages/plugin-patch/sources/commands/patch.ts
@@ -1,9 +1,9 @@
-import {BaseCommand, WorkspaceRequiredError}                                                from '@yarnpkg/cli';
-import {Cache, Configuration, Project, ThrowReport, structUtils, StreamReport, MessageName} from '@yarnpkg/core';
-import {npath}                                                                              from '@yarnpkg/fslib';
-import {Command, Usage, UsageError}                                                         from 'clipanion';
+import {BaseCommand, WorkspaceRequiredError}                                   from '@yarnpkg/cli';
+import {Cache, Configuration, Project, structUtils, StreamReport, MessageName} from '@yarnpkg/core';
+import {npath}                                                                 from '@yarnpkg/fslib';
+import {Command, Usage, UsageError}                                            from 'clipanion';
 
-import * as patchUtils                                                                      from '../patchUtils';
+import * as patchUtils                                                         from '../patchUtils';
 
 // eslint-disable-next-line arca/no-default-export
 export default class PatchCommand extends BaseCommand {
@@ -25,10 +25,7 @@ export default class PatchCommand extends BaseCommand {
     if (!workspace)
       throw new WorkspaceRequiredError(project.cwd, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     let locator = structUtils.parseLocator(this.package);
 

diff --git a/packages/plugin-patch/sources/commands/patchCommit.ts b/packages/plugin-patch/sources/commands/patchCommit.ts
@@ -1,9 +1,9 @@
-import {BaseCommand, WorkspaceRequiredError}                                           from '@yarnpkg/cli';
-import {Cache, Configuration, Project, ThrowReport, structUtils, execUtils, miscUtils} from '@yarnpkg/core';
-import {npath, xfs, Filename, ppath}                                                   from '@yarnpkg/fslib';
-import {Command, Usage, UsageError}                                                    from 'clipanion';
+import {BaseCommand, WorkspaceRequiredError}                              from '@yarnpkg/cli';
+import {Cache, Configuration, Project, structUtils, execUtils, miscUtils} from '@yarnpkg/core';
+import {npath, xfs, Filename, ppath}                                      from '@yarnpkg/fslib';
+import {Command, Usage, UsageError}                                       from 'clipanion';
 
-import * as patchUtils                                                                 from '../patchUtils';
+import * as patchUtils                                                    from '../patchUtils';
 
 // eslint-disable-next-line arca/no-default-export
 export default class PatchCommitCommand extends BaseCommand {
@@ -27,10 +27,7 @@ export default class PatchCommitCommand extends BaseCommand {
     if (!workspace)
       throw new WorkspaceRequiredError(project.cwd, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     const folderPath = ppath.resolve(this.context.cwd, npath.toPortablePath(this.patchFolder));
     const metaPath = ppath.join(folderPath, `.yarn-patch.json` as Filename);

diff --git a/packages/plugin-version/sources/commands/version/check.tsx b/packages/plugin-version/sources/commands/version/check.tsx
@@ -1,16 +1,16 @@
-import {WorkspaceRequiredError}                                                                                 from '@yarnpkg/cli';
-import {CommandContext, Configuration, MessageName, Project, StreamReport, Workspace, structUtils, ThrowReport} from '@yarnpkg/core';
-import {ppath}                                                                                                  from '@yarnpkg/fslib';
-import {ScrollableItems}                                                                                        from '@yarnpkg/libui/sources/components/ScrollableItems';
-import {FocusRequest}                                                                                           from '@yarnpkg/libui/sources/hooks/useFocusRequest';
-import {useListInput}                                                                                           from '@yarnpkg/libui/sources/hooks/useListInput';
-import {renderForm}                                                                                             from '@yarnpkg/libui/sources/misc/renderForm';
-import {Command, Usage, UsageError}                                                                             from 'clipanion';
-import {Box, Color}                                                                                             from 'ink';
-import React, {useCallback, useState}                                                                           from 'react';
-import semver                                                                                                   from 'semver';
-
-import * as versionUtils                                                                                        from '../../versionUtils';
+import {WorkspaceRequiredError}                                                                    from '@yarnpkg/cli';
+import {CommandContext, Configuration, MessageName, Project, StreamReport, Workspace, structUtils} from '@yarnpkg/core';
+import {ppath}                                                                                     from '@yarnpkg/fslib';
+import {ScrollableItems}                                                                           from '@yarnpkg/libui/sources/components/ScrollableItems';
+import {FocusRequest}                                                                              from '@yarnpkg/libui/sources/hooks/useFocusRequest';
+import {useListInput}                                                                              from '@yarnpkg/libui/sources/hooks/useListInput';
+import {renderForm}                                                                                from '@yarnpkg/libui/sources/misc/renderForm';
+import {Command, Usage, UsageError}                                                                from 'clipanion';
+import {Box, Color}                                                                                from 'ink';
+import React, {useCallback, useState}                                                              from 'react';
+import semver                                                                                      from 'semver';
+
+import * as versionUtils                                                                           from '../../versionUtils';
 
 type Releases = Map<Workspace, Exclude<versionUtils.Decision, versionUtils.Decision.UNDECIDED>>;
 
@@ -53,10 +53,7 @@ export default class VersionApplyCommand extends Command<CommandContext> {
     if (!workspace)
       throw new WorkspaceRequiredError(project.cwd, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     const versionFile = await versionUtils.openVersionFile(project);
     if (versionFile === null || versionFile.releaseRoots.size === 0)
@@ -273,10 +270,7 @@ export default class VersionApplyCommand extends Command<CommandContext> {
     if (!workspace)
       throw new WorkspaceRequiredError(project.cwd, this.context.cwd);
 
-    await project.resolveEverything({
-      lockfileOnly: true,
-      report: new ThrowReport(),
-    });
+    await project.restoreInstallState();
 
     const report = await StreamReport.start({
       configuration,

diff --git a/packages/yarnpkg-core/package.json b/packages/yarnpkg-core/package.json
@@ -37,6 +37,7 @@
     "@types/got": "^8.3.4",
     "@types/is-ci": "^2.0.0",
     "@types/micromatch": "^3.1.0",
+    "@types/node": "^13.7.0",
     "@types/semver": "^7.1.0",
     "@types/tar": "^4.0.0",
     "@types/tmp": "^0.0.33",

diff --git a/packages/yarnpkg-core/sources/Configuration.ts b/packages/yarnpkg-core/sources/Configuration.ts
@@ -195,6 +195,11 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     type: SettingsType.STRING,
     default: DEFAULT_LOCK_FILENAME,
   },
+  installStatePath: {
+    description: `Path of the file where the install state will be persisted`,
+    type: SettingsType.ABSOLUTE_PATH,
+    default: `./.yarn/install-state.gz`,
+  },
   rcFilename: {
     description: `Name of the files where the configuration can be found`,
     type: SettingsType.STRING,