Async libzip (yarnpkg#731)

* wip

* Implements asynchronous libzip

* Adds versions

* Removes the dependency on process.binding

* Fixes PatchFetcher

* Fixes imports

* Pins workflow to 13.5

* Pins to 13.1

.eslintignore

@@ -0,0 +1,2 @@
+/packages/yarnpkg-libzip/sources/libzipAsync.js
+/packages/yarnpkg-libzip/sources/libzipSync.js

.github/workflows/integration-workflow.yml

@@ -62,7 +62,7 @@ jobs:
         node:
         - 10
         - 12
-        - 13
+        - 13.1 # https://github.com/nodejs/node/issues/30846 (libzip crashes on every platform on >13.1)
         platform:
         - ubuntu-latest
         - windows-latest

.yarn/versions/8fb3b16d.yml

@@ -0,0 +1,36 @@
+releases:
+  "@yarnpkg/core": prerelease
+  "@yarnpkg/fslib": prerelease
+  "@yarnpkg/libzip": prerelease
+  "@yarnpkg/plugin-node-modules": prerelease
+  "@yarnpkg/plugin-patch": prerelease
+  "@yarnpkg/pnp": prerelease
+  vscode-zipfs: prerelease
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/check"
+  - "@yarnpkg/cli"
+  - "@yarnpkg/json-proxy"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/shell"

jest.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   testEnvironment: require.resolve(`jest-environment-node`),
-  transformIgnorePatterns: [`${__dirname}/packages/yarnpkg-libzip/sources/libzip.js$`, `/.pnp.js$`],
+  transformIgnorePatterns: [`${__dirname}/packages/yarnpkg-libzip/sources/libzip(Async|Sync).js$`, `/.pnp.js$`],
   modulePathIgnorePatterns: [`<rootDir>/packages/acceptance-tests`, `packages/gatsby/.cache`],
   reporters: [`default`, [require.resolve(`jest-junit`), {output: `<rootDir>/junit.xml`}]],
   setupFiles: [`@yarnpkg/cli/sources/polyfills.ts`],

packages/plugin-node-modules/package.json

@@ -6,6 +6,7 @@
     "@yarnpkg/cli": "workspace:^2.0.0-rc.24",
     "@yarnpkg/core": "workspace:^2.0.0-rc.19",
     "@yarnpkg/fslib": "workspace:^2.0.0-rc.12",
+    "@yarnpkg/libzip": "workspace:^2.0.0-rc.5",
     "@yarnpkg/parsers": "workspace:^2.0.0-rc.8",
     "@yarnpkg/plugin-pnp": "workspace:^2.0.0-rc.14",
     "@yarnpkg/pnp": "workspace:^2.0.0-rc.15",

packages/plugin-node-modules/sources/NodeModulesLinker.ts

@@ -1,9 +1,10 @@
-import {BuildDirective, MessageName, Project}                                                         from '@yarnpkg/core';
-import {Linker, LinkOptions, MinimalLinkOptions, LinkType}                                            from '@yarnpkg/core';
-import {Locator, Package, BuildType}                                                                  from '@yarnpkg/core';
 import {structUtils, Report, Manifest, miscUtils, FinalizeInstallStatus, FetchResult, DependencyMeta} from '@yarnpkg/core';
-import {VirtualFS, ZipOpenFS}                                                                         from '@yarnpkg/fslib';
+import {Locator, Package, BuildType}                                                                  from '@yarnpkg/core';
+import {Linker, LinkOptions, MinimalLinkOptions, LinkType}                                            from '@yarnpkg/core';
+import {BuildDirective, MessageName, Project}                                                         from '@yarnpkg/core';
 import {PortablePath, npath, ppath, toFilename, Filename, xfs, FakeFS}                                from '@yarnpkg/fslib';
+import {VirtualFS, ZipOpenFS}                                                                         from '@yarnpkg/fslib';
+import {getLibzipPromise}                                                                             from '@yarnpkg/libzip';
 import {parseSyml}                                                                                    from '@yarnpkg/parsers';
 import {AbstractPnpInstaller}                                                                         from '@yarnpkg/plugin-pnp';
 import {NodeModulesLocatorMap, buildLocatorMap, buildNodeModulesTree}                                 from '@yarnpkg/pnpify';
@@ -72,6 +73,7 @@ class NodeModulesInstaller extends AbstractPnpInstaller {
 
     const defaultFsLayer = new VirtualFS({
       baseFs: new ZipOpenFS({
+        libzip: await getLibzipPromise(),
         maxOpenFiles: 80,
         readOnlyArchives: true,
       }),

packages/plugin-patch/package.json

@@ -4,6 +4,7 @@
   "main": "./sources/index.ts",
   "dependencies": {
     "@yarnpkg/fslib": "workspace:^2.0.0-rc.12",
+    "@yarnpkg/libzip": "workspace:^2.0.0-rc.5",
     "clipanion": "^2.1.5"
   },
   "peerDependencies": {

packages/plugin-patch/sources/PatchFetcher.ts

@@ -2,6 +2,7 @@ import {Fetcher, FetchOptions, MinimalFetchOptions} from '@yarnpkg/core';
 import {Locator, MessageName}                       from '@yarnpkg/core';
 import {miscUtils, structUtils}                     from '@yarnpkg/core';
 import {ppath, xfs, ZipFS, Filename, CwdFS}         from '@yarnpkg/fslib';
+import {getLibzipPromise}                           from '@yarnpkg/libzip';
 
 import * as patchUtils                              from './patchUtils';
 
@@ -48,7 +49,9 @@ export class PatchFetcher implements Fetcher {
     const sourceFetch = await opts.fetcher.fetch(sourceLocator, opts);
     const prefixPath = structUtils.getIdentVendorPath(locator);
 
-    const copiedPackage = new ZipFS(tmpFile, {create: true});
+    const libzip = await getLibzipPromise();
+
+    const copiedPackage = new ZipFS(tmpFile, {libzip, create: true});
     await copiedPackage.mkdirpPromise(prefixPath);
 
     await miscUtils.releaseAfterUseAsync(async () => {
@@ -57,7 +60,7 @@ export class PatchFetcher implements Fetcher {
 
     copiedPackage.saveAndClose();
 
-    const patchedPackage = new ZipFS(tmpFile);
+    const patchedPackage = new ZipFS(tmpFile, {libzip});
     const patchFs = new CwdFS(prefixPath, {baseFs: patchedPackage});
 
     for (const patchFile of patchFiles) {

packages/vscode-zipfs/package.json

@@ -64,6 +64,7 @@
   },
   "devDependencies": {
     "@yarnpkg/fslib": "workspace:^2.0.0-rc.12",
+    "@yarnpkg/libzip": "workspace:^2.0.0-rc.5",
     "@yarnpkg/pnpify": "workspace:^2.0.0-rc.15",
     "pnp-webpack-plugin": "^1.4.3",
     "ts-loader": "^6.2.1",

packages/vscode-zipfs/sources/ZipFSProvider.ts

@@ -1,9 +1,13 @@
 import {ZipOpenFS, PortablePath} from '@yarnpkg/fslib';
+import {getLibzipSync}           from '@yarnpkg/libzip';
 import {posix}                   from 'path';
 import * as vscode               from 'vscode';
 
 export class ZipFSProvider implements vscode.FileSystemProvider {
-  private readonly zipFs = new ZipOpenFS({useCache: false});
+  private readonly zipFs = new ZipOpenFS({
+    libzip: getLibzipSync(),
+    useCache: false,
+  });
 
   stat(uri: vscode.Uri): vscode.FileStat {
     const stat: any = this.zipFs.statSync(uri.path as PortablePath);

packages/yarnpkg-core/package.json

@@ -6,6 +6,7 @@
   "dependencies": {
     "@yarnpkg/fslib": "workspace:^2.0.0-rc.12",
     "@yarnpkg/json-proxy": "workspace:^2.0.0-rc.4",
+    "@yarnpkg/libzip": "workspace:^2.0.0-rc.5",
     "@yarnpkg/parsers": "workspace:^2.0.0-rc.8",
     "@yarnpkg/pnp": "workspace:^2.0.0-rc.15",
     "@yarnpkg/shell": "workspace:^2.0.0-rc.4",

packages/yarnpkg-core/sources/Cache.ts

@@ -1,12 +1,14 @@
 import {FakeFS, LazyFS, NodeFS, ZipFS, PortablePath, Filename} from '@yarnpkg/fslib';
 import {npath, ppath, toFilename, xfs}                         from '@yarnpkg/fslib';
+import {getLibzipPromise}                                      from '@yarnpkg/libzip';
 import fs                                                      from 'fs';
 import {tmpNameSync}                                           from 'tmp';
 
 import {Configuration}                                         from './Configuration';
 import {MessageName}                                           from './MessageName';
 import {ReportError}                                           from './Report';
 import * as hashUtils                                          from './hashUtils';
+import * as miscUtils                                          from './miscUtils';
 import * as structUtils                                        from './structUtils';
 import {LocatorHash, Locator}                                  from './types';
 
@@ -149,7 +151,10 @@ export class Cache {
 
       const tempPath = npath.toPortablePath(tmpNameSync());
       await xfs.copyFilePromise(mirrorPath, tempPath, fs.constants.COPYFILE_FICLONE);
-      return new ZipFS(tempPath);
+
+      return new ZipFS(tempPath, {
+        libzip: await getLibzipPromise(),
+      });
     };
 
     const loadPackage = async () => {
@@ -203,14 +208,12 @@ export class Cache {
 
     let zipFs: ZipFS | null = null;
 
-    const lazyFs: LazyFS<PortablePath> = new LazyFS<PortablePath>(() => {
-      try {
-        return zipFs = new ZipFS(cachePath, {readOnly: true, baseFs});
-      } catch (error) {
-        error.message = `Failed to open the cache entry for ${structUtils.prettyLocator(this.configuration, locator)}: ${error.message}`;
-        throw error;
-      }
-    }, ppath);
+    const libzip = await getLibzipPromise();
+    const lazyFs: LazyFS<PortablePath> = new LazyFS<PortablePath>(() => miscUtils.prettifySyncErrors(() => {
+      return zipFs = new ZipFS(cachePath, {baseFs, libzip, readOnly: true});
+    }, message => {
+      return `Failed to open the cache entry for ${structUtils.prettyLocator(this.configuration, locator)}: ${message}`;
+    }), ppath);
 
     const releaseFs = () => {
       if (zipFs !== null) {

packages/yarnpkg-core/sources/scriptUtils.ts

@@ -1,5 +1,6 @@
 import {CwdFS, Filename, NativePath, PortablePath, ZipOpenFS} from '@yarnpkg/fslib';
 import {xfs, npath, ppath, toFilename}                        from '@yarnpkg/fslib';
+import {getLibzipPromise}                                     from '@yarnpkg/libzip';
 import {execute}                                              from '@yarnpkg/shell';
 import {PassThrough, Readable, Writable}                      from 'stream';
 import {dirSync}                                              from 'tmp';
@@ -126,6 +127,8 @@ export async function hasPackageScript(locator: Locator, scriptName: string, {pr
     const manifest = await Manifest.find(PortablePath.dot, {baseFs: packageFs});
 
     return manifest.scripts.has(scriptName);
+  }, {
+    libzip: await getLibzipPromise(),
   });
 }
 
@@ -200,6 +203,8 @@ async function initializePackageEnvironment(locator: Locator, {project, cwd, lif
       cwd = packageLocation;
 
     return {manifest, binFolder, env, cwd};
+  }, {
+    libzip: await getLibzipPromise(),
   });
 }
 

packages/yarnpkg-core/sources/tgzUtils.ts

@@ -1,4 +1,5 @@
 import {FakeFS, PortablePath, ZipFS, NodeFS, npath, ppath} from '@yarnpkg/fslib';
+import {getLibzipPromise}                                  from '@yarnpkg/libzip';
 import {Parse}                                             from 'tar';
 import {tmpNameSync}                                       from 'tmp';
 
@@ -8,7 +9,7 @@ interface MakeArchiveFromDirectoryOptions {
 };
 
 export async function makeArchiveFromDirectory(source: PortablePath, {baseFs = new NodeFS(), prefixPath = PortablePath.root}: MakeArchiveFromDirectoryOptions = {}): Promise<ZipFS> {
-  const zipFs = new ZipFS(npath.toPortablePath(tmpNameSync()), {create: true});
+  const zipFs = new ZipFS(npath.toPortablePath(tmpNameSync()), {create: true, libzip: await getLibzipPromise()});
   const target = ppath.resolve(PortablePath.root, prefixPath!);
 
   await zipFs.copyPromise(target, source, {baseFs});
@@ -22,7 +23,7 @@ interface ExtractBufferOptions {
 };
 
 export async function convertToZip(tgz: Buffer, opts: ExtractBufferOptions) {
-  return await extractArchiveTo(tgz, new ZipFS(npath.toPortablePath(tmpNameSync()), {create: true}), opts);
+  return await extractArchiveTo(tgz, new ZipFS(npath.toPortablePath(tmpNameSync()), {create: true, libzip: await getLibzipPromise()}), opts);
 }
 
 export async function extractArchiveTo<T extends FakeFS<PortablePath>>(tgz: Buffer, targetFs: T, {stripComponents = 0, prefixPath = PortablePath.dot}: ExtractBufferOptions = {}): Promise<T> {