feat: unplug packages with native files (yarnpkg#853)

* feat: unplug packages with native files * chore: lint * chore: version check * chore: remove debugging leftovers * chore: update pnp map * fix(pnpify): add getExtractHint * chore: version check * chore: fix linting
bertho-zero · Jan 30, 2020 · 32d620f · 32d620f
1 parent 34423f3
commit 32d620f
Show file tree

Hide file tree

Showing 14 changed files with 123 additions and 34 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -27,6 +27,7 @@
     "comma-dangle": ["error", "always-multiline"],
     "computed-property-spacing": 2,
     "generator-star-spacing": ["error", {"before": true, "after": true}],
+    "space-before-blocks": 2,
     "keyword-spacing": 2,
     "no-multiple-empty-lines": 2,
     "no-tabs": 2,

diff --git a/.pnp.js b/.pnp.js
diff --git a/.yarn/versions/325ffac3.yml b/.yarn/versions/325ffac3.yml
@@ -0,0 +1,32 @@
+releases:
+  "@yarnpkg/fslib": prerelease
+  "@yarnpkg/plugin-pnp": prerelease
+  "@yarnpkg/pnpify": prerelease
+
+declined:
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-node-modules"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/cli"
+  - "@yarnpkg/core"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/json-proxy"
+  - "@yarnpkg/pnp"
+  - "@yarnpkg/shell"
diff --git a/packages/plugin-pnp/sources/PnpLinker.ts b/packages/plugin-pnp/sources/PnpLinker.ts
@@ -18,6 +18,17 @@ const FORCED_UNPLUG_PACKAGES = new Set([
   structUtils.makeIdent(null, `fsevents`).identHash,
 ]);
 
+const FORCED_UNPLUG_FILETYPES = new Set([
+  // Windows can't execute exe files inside zip archives
+  '.exe',
+  // The c/c++ compiler can't read files from zip archives
+  '.h', '.hh', '.hpp', '.c', '.cc', '.cpp',
+  // The java runtime can't read files from zip archives
+  '.java',
+  // Node opens these through dlopen
+  '.node',
+]);
+
 export class PnpLinker implements Linker {
   supportsPackage(pkg: Package, opts: MinimalLinkOptions) {
     return opts.project.configuration.get('nodeLinker') === 'pnp';
@@ -83,7 +94,7 @@ class PnpInstaller extends AbstractPnpInstaller {
   }
 
   async transformPackage(locator: Locator, dependencyMeta: DependencyMeta, packageFs: FakeFS<PortablePath>, {hasBuildScripts}: {hasBuildScripts: boolean}) {
-    if (hasBuildScripts || this.isUnplugged(locator, dependencyMeta)) {
+    if (hasBuildScripts || this.isUnplugged(locator, dependencyMeta, packageFs)) {
       return this.unplugPackage(locator, packageFs);
     } else {
       return packageFs;
@@ -184,13 +195,16 @@ class PnpInstaller extends AbstractPnpInstaller {
     return new CwdFS(unplugPath);
   }
 
-  private isUnplugged(ident: Ident, dependencyMeta: DependencyMeta) {
+  private isUnplugged(ident: Ident, dependencyMeta: DependencyMeta, packageFs: FakeFS<PortablePath>) {
     if (dependencyMeta.unplugged)
       return true;
 
     if (FORCED_UNPLUG_PACKAGES.has(ident.identHash))
       return true;
 
+    if (packageFs.getExtractHint({relevantExtensions:FORCED_UNPLUG_FILETYPES}))
+      return true;
+
     return false;
   }
 }
diff --git a/packages/yarnpkg-fslib/sources/FakeFS.ts b/packages/yarnpkg-fslib/sources/FakeFS.ts
@@ -51,6 +51,10 @@ export type Watcher = {
   close: () => void;
 };
 
+export type ExtractHintOptions = {
+  relevantExtensions: Set<string>;
+}
+
 export abstract class FakeFS<P extends Path> {
   static DEFAULT_TIME = 315532800;
 
@@ -60,6 +64,8 @@ export abstract class FakeFS<P extends Path> {
     this.pathUtils =  pathUtils;
   }
 
+  abstract getExtractHint(hints: ExtractHintOptions): boolean;
+
   abstract getRealPath(): P;
 
   abstract resolve(p: P): P;
@@ -528,6 +534,6 @@ function getEndOfLine(content: string) {
   return crlf > lf ? `\r\n` : `\n`;
 }
 
-export function normalizeLineEndings(originalContent: string, newContent: string){
+export function normalizeLineEndings(originalContent: string, newContent: string) {
   return newContent.replace(/\r?\n/g, getEndOfLine(originalContent));
 }
diff --git a/packages/yarnpkg-fslib/sources/NoFS.ts b/packages/yarnpkg-fslib/sources/NoFS.ts
@@ -10,6 +10,10 @@ export class NoFS extends FakeFS<PortablePath> {
     super(ppath);
   }
 
+  getExtractHint(): never {
+    throw makeError();
+  }
+
   getRealPath(): never {
     throw makeError();
   }

diff --git a/packages/yarnpkg-fslib/sources/NodeFS.ts b/packages/yarnpkg-fslib/sources/NodeFS.ts
@@ -15,6 +15,10 @@ export class NodeFS extends BasePortableFakeFS {
     this.realFs = realFs;
   }
 
+  getExtractHint() {
+    return false;
+  }
+
   getRealPath() {
     return PortablePath.root;
   }

diff --git a/packages/yarnpkg-fslib/sources/ProxiedFS.ts b/packages/yarnpkg-fslib/sources/ProxiedFS.ts
@@ -1,7 +1,7 @@
-import {CreateReadStreamOptions, CreateWriteStreamOptions, FakeFS}            from './FakeFS';
-import {Dirent}                                                               from './FakeFS';
-import {MkdirOptions, WriteFileOptions, WatchCallback, WatchOptions, Watcher} from './FakeFS';
-import {FSPath, Filename, Path}                                               from './path';
+import {CreateReadStreamOptions, CreateWriteStreamOptions, FakeFS, ExtractHintOptions} from './FakeFS';
+import {Dirent}                                                                        from './FakeFS';
+import {MkdirOptions, WriteFileOptions, WatchCallback, WatchOptions, Watcher}          from './FakeFS';
+import {FSPath, Filename, Path}                                                        from './path';
 
 export abstract class ProxiedFS<P extends Path, IP extends Path> extends FakeFS<P> {
   protected abstract readonly baseFs: FakeFS<IP>;
@@ -16,6 +16,10 @@ export abstract class ProxiedFS<P extends Path, IP extends Path> extends FakeFS<
    */
   protected abstract mapFromBase(path: IP): P;
 
+  getExtractHint(hints: ExtractHintOptions) {
+    return this.baseFs.getExtractHint(hints);
+  }
+
   resolve(path: P)  {
     return this.mapFromBase(this.baseFs.resolve(this.mapToBase(path)));
   }

diff --git a/packages/yarnpkg-fslib/sources/VirtualFS.ts b/packages/yarnpkg-fslib/sources/VirtualFS.ts
@@ -1,4 +1,4 @@
-import {FakeFS}                        from './FakeFS';
+import {FakeFS, ExtractHintOptions}    from './FakeFS';
 import {NodeFS}                        from './NodeFS';
 import {ProxiedFS}                     from './ProxiedFS';
 import {Filename, PortablePath, ppath} from './path';
@@ -66,6 +66,10 @@ export class VirtualFS extends ProxiedFS<PortablePath, PortablePath> {
     this.baseFs = baseFs;
   }
 
+  getExtractHint(hints: ExtractHintOptions) {
+    return this.baseFs.getExtractHint(hints);
+  }
+
   getRealPath() {
     return this.baseFs.getRealPath();
   }

diff --git a/packages/yarnpkg-fslib/sources/ZipFS.ts b/packages/yarnpkg-fslib/sources/ZipFS.ts
@@ -1,14 +1,14 @@
-import {Libzip}                                                                from '@yarnpkg/libzip';
-import {ReadStream, Stats, WriteStream, constants}                             from 'fs';
-import {PassThrough}                                                           from 'stream';
-import {isDate}                                                                from 'util';
-
-import {CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS} from './FakeFS';
-import {FakeFS, MkdirOptions, WriteFileOptions}                                from './FakeFS';
-import {WatchOptions, WatchCallback, Watcher}                                  from './FakeFS';
-import {NodeFS}                                                                from './NodeFS';
-import * as errors                                                             from './errors';
-import {FSPath, PortablePath, npath, ppath, Filename}                          from './path';
+import {Libzip}                                                                                    from '@yarnpkg/libzip';
+import {ReadStream, Stats, WriteStream, constants}                                                 from 'fs';
+import {PassThrough}                                                                               from 'stream';
+import {isDate}                                                                                    from 'util';
+
+import {CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS, ExtractHintOptions} from './FakeFS';
+import {FakeFS, MkdirOptions, WriteFileOptions}                                                    from './FakeFS';
+import {WatchOptions, WatchCallback, Watcher}                                                      from './FakeFS';
+import {NodeFS}                                                                                    from './NodeFS';
+import * as errors                                                                                 from './errors';
+import {FSPath, PortablePath, npath, ppath, Filename}                                              from './path';
 
 const S_IFMT = 0o170000;
 
@@ -251,6 +251,17 @@ export class ZipFS extends BasePortableFakeFS {
     this.ready = true;
   }
 
+  getExtractHint(hints: ExtractHintOptions) {
+    for (const fileName of this.entries.keys()) {
+      const ext = this.pathUtils.extname(fileName);
+      if (hints.relevantExtensions.has(ext)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
   getAllFiles() {
     return Array.from(this.entries.keys());
   }

diff --git a/packages/yarnpkg-fslib/sources/ZipOpenFS.ts b/packages/yarnpkg-fslib/sources/ZipOpenFS.ts
@@ -1,13 +1,13 @@
-import {Libzip}                                                                from '@yarnpkg/libzip';
-import {constants}                                                             from 'fs';
+import {Libzip}                                                                                    from '@yarnpkg/libzip';
+import {constants}                                                                                 from 'fs';
 
-import {CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS} from './FakeFS';
-import {Dirent}                                                                from './FakeFS';
-import {FakeFS, MkdirOptions, WriteFileOptions}                                from './FakeFS';
-import {WatchOptions, WatchCallback, Watcher}                                  from './FakeFS';
-import {NodeFS}                                                                from './NodeFS';
-import {ZipFS}                                                                 from './ZipFS';
-import {Filename, FSPath, PortablePath}                                        from './path';
+import {CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS, ExtractHintOptions} from './FakeFS';
+import {Dirent}                                                                                    from './FakeFS';
+import {FakeFS, MkdirOptions, WriteFileOptions}                                                    from './FakeFS';
+import {WatchOptions, WatchCallback, Watcher}                                                      from './FakeFS';
+import {NodeFS}                                                                                    from './NodeFS';
+import {ZipFS}                                                                                     from './ZipFS';
+import {Filename, FSPath, PortablePath}                                                            from './path';
 
 const ZIP_FD = 0x80000000;
 
@@ -64,6 +64,10 @@ export class ZipOpenFS extends BasePortableFakeFS {
     this.notZip = new Set();
   }
 
+  getExtractHint(hints: ExtractHintOptions) {
+    return this.baseFs.getExtractHint(hints);
+  }
+
   getRealPath() {
     return this.baseFs.getRealPath();
   }

diff --git a/packages/yarnpkg-fslib/sources/index.ts b/packages/yarnpkg-fslib/sources/index.ts
@@ -13,6 +13,7 @@ export {WatchCallback}            from './FakeFS';
 export {Watcher}                  from './FakeFS';
 export {WriteFileOptions}         from './FakeFS';
 export {normalizeLineEndings}     from './FakeFS';
+export {ExtractHintOptions}       from './FakeFS';
 
 export {FSPath, Path, PortablePath, NativePath, Filename} from './path';
 export {ParsedPath, PathUtils, FormatInputPathObject} from './path';

diff --git a/packages/yarnpkg-pnp/sources/generateSerializedState.ts b/packages/yarnpkg-pnp/sources/generateSerializedState.ts
@@ -1,5 +1,5 @@
-import {LocationBlacklistData, LocationLengthData, PackageRegistryData} from './types';
-import {PackageStoreData, PnpSettings, SerializedState}                 from './types';
+import {LocationBlacklistData, PackageRegistryData}     from './types';
+import {PackageStoreData, PnpSettings, SerializedState} from './types';
 
 // Keep this function is sync with its implementation in:
 // @yarnpkg/core/sources/miscUtils.ts

diff --git a/packages/yarnpkg-pnpify/sources/NodeModulesFS.ts b/packages/yarnpkg-pnpify/sources/NodeModulesFS.ts
@@ -1,4 +1,4 @@
-import {Dirent, Filename, MkdirOptions}                    from '@yarnpkg/fslib';
+import {Dirent, Filename, MkdirOptions,ExtractHintOptions} from '@yarnpkg/fslib';
 import {FSPath, NativePath, PortablePath, npath, ppath}    from '@yarnpkg/fslib';
 import {WatchOptions, WatchCallback, Watcher, toFilename}  from '@yarnpkg/fslib';
 import {NodeFS, FakeFS, WriteFileOptions, ProxiedFS}       from '@yarnpkg/fslib';
@@ -86,6 +86,10 @@ export class PortableNodeModulesFS extends FakeFS<PortablePath> {
     }
   }
 
+  getExtractHint(hints: ExtractHintOptions) {
+    return this.baseFs.getExtractHint(hints);
+  }
+
   resolve(path: PortablePath) {
     return this.baseFs.resolve(this.resolvePath(path).resolvedPath!);
   }