feat: add node-pre-gyp support (#1095)

* add NodePreGyp module * chore: update node-abi * fix: node-pre-gyp args * feat: add napi support * test: add initial node-pre-gyp tests * fix: some tests * test: skip on m1 * chore: remove unused code * feat: add skipPrebuilds option * fix: napi build version test * fix: spawning node-pre-gyp on windows * use fs.rmdir maxRetries Co-authored-by: Keeley Hammond <[email protected]> * rename 'skipPreloads' to 'buildFromSource' --------- Co-authored-by: George Xu <[email protected]> Co-authored-by: Keeley Hammond <[email protected]>
electron · Aug 21, 2023 · e71316b · e71316b
1 parent 6b485d6
commit e71316b
Show file tree

Hide file tree

Showing 13 changed files with 167 additions and 14 deletions.
diff --git a/package.json b/package.json
@@ -45,7 +45,7 @@
     "detect-libc": "^2.0.1",
     "fs-extra": "^10.0.0",
     "got": "^11.7.0",
-    "node-abi": "^3.0.0",
+    "node-abi": "^3.45.0",
     "node-api-version": "^0.1.4",
     "node-gyp": "^9.0.0",
     "ora": "^5.1.0",

diff --git a/src/cli.ts b/src/cli.ts
@@ -128,6 +128,7 @@ process.on('unhandledRejection', handler);
     useElectronClang: !!argv.useElectronClang,
     disablePreGypCopy: !!argv.disablePreGypCopy,
     projectRootPath,
+    buildFromSource: !!argv.buildFromSource,
   });
 
   const lifecycle = rebuilder.lifecycle;

diff --git a/src/module-rebuilder.ts b/src/module-rebuilder.ts
@@ -6,6 +6,7 @@ import { cacheModuleState } from './cache';
 import { NodeGyp } from './module-type/node-gyp/node-gyp';
 import { Prebuildify } from './module-type/prebuildify';
 import { PrebuildInstall } from './module-type/prebuild-install';
+import { NodePreGyp } from './module-type/node-pre-gyp';
 import { IRebuilder } from './types';
 
 const d = debug('electron-rebuild');
@@ -16,6 +17,7 @@ export class ModuleRebuilder {
   private rebuilder: IRebuilder;
   private prebuildify: Prebuildify;
   private prebuildInstall: PrebuildInstall;
+  private nodePreGyp: NodePreGyp;
 
   constructor(rebuilder: IRebuilder, modulePath: string) {
     this.modulePath = modulePath;
@@ -24,6 +26,7 @@ export class ModuleRebuilder {
     this.nodeGyp = new NodeGyp(rebuilder, modulePath);
     this.prebuildify = new Prebuildify(rebuilder, modulePath);
     this.prebuildInstall = new PrebuildInstall(rebuilder, modulePath);
+    this.nodePreGyp = new NodePreGyp(rebuilder, modulePath);
   }
 
   get metaPath(): string {
@@ -89,6 +92,21 @@ export class ModuleRebuilder {
     return false;
   }
 
+  async findNodePreGypInstallModule(cacheKey: string): Promise<boolean> {
+    if (await this.nodePreGyp.usesTool()) {
+      d(`assuming is node-pre-gyp powered: ${this.nodePreGyp.moduleName}`);
+
+      if (await this.nodePreGyp.findPrebuiltModule()) {
+        d('installed prebuilt module:', this.nodePreGyp.moduleName);
+        await this.writeMetadata();
+        await this.cacheModuleState(cacheKey);
+        return true;
+      }
+    }
+
+    return false;
+  }
+
   async rebuildNodeGypModule(cacheKey: string): Promise<boolean> {
     await this.nodeGyp.rebuildModule();
     d('built via node-gyp:', this.nodeGyp.moduleName);
@@ -124,8 +142,15 @@ export class ModuleRebuilder {
   }
 
   async rebuild(cacheKey: string): Promise<boolean> {
-    return (await this.findPrebuildifyModule(cacheKey)) ||
-      (await this.findPrebuildInstallModule(cacheKey)) ||
-      (await this.rebuildNodeGypModule(cacheKey));
+    if (
+      !this.rebuilder.buildFromSource && (
+        (await this.findPrebuildifyModule(cacheKey)) ||
+        (await this.findPrebuildInstallModule(cacheKey)) ||
+        (await this.findNodePreGypInstallModule(cacheKey)))
+    ) {
+      return true;
+    }
+
+    return await this.rebuildNodeGypModule(cacheKey);
   }
 }
diff --git a/src/module-type/node-pre-gyp.ts b/src/module-type/node-pre-gyp.ts
@@ -0,0 +1,68 @@
+import debug from 'debug';
+import { spawn } from '@malept/cross-spawn-promise';
+
+import { locateBinary, NativeModule } from '.';
+const d = debug('electron-rebuild');
+
+export class NodePreGyp extends NativeModule {
+  async usesTool(): Promise<boolean> {
+    const dependencies = await this.packageJSONFieldWithDefault('dependencies', {});
+    // eslint-disable-next-line no-prototype-builtins
+    return dependencies.hasOwnProperty('@mapbox/node-pre-gyp');
+  }
+
+  async locateBinary(): Promise<string | null> {
+    return locateBinary(this.modulePath, 'node_modules/@mapbox/node-pre-gyp/bin/node-pre-gyp');
+  }
+
+  async run(nodePreGypPath: string): Promise<void> {
+    await spawn(
+      process.execPath,
+      [
+        nodePreGypPath,
+        'reinstall',
+        '--fallback-to-build',
+        `--arch=${this.rebuilder.arch}`,
+        `--platform=${this.rebuilder.platform}`,
+        ...await this.getNodePreGypRuntimeArgs(),
+      ],
+      {
+        cwd: this.modulePath,
+      }
+    );
+  }
+
+  async findPrebuiltModule(): Promise<boolean> {
+    const nodePreGypPath = await this.locateBinary();
+    if (nodePreGypPath) {
+      d(`triggering prebuild download step: ${this.moduleName}`);
+      try {
+        await this.run(nodePreGypPath);
+        return true;
+      } catch (err) {
+        d('failed to use node-pre-gyp:', err);
+
+        if (err?.message?.includes('requires Node-API but Electron')) {
+          throw err;
+        }
+      }
+    } else {
+      d(`could not find node-pre-gyp relative to: ${this.modulePath}`);
+    }
+
+    return false;
+  }
+
+  async getNodePreGypRuntimeArgs(): Promise<string[]> {
+    const moduleNapiVersions = await this.getSupportedNapiVersions();
+    if (moduleNapiVersions) {
+      return [];
+    } else {
+      return [
+        '--runtime=electron',
+        `--target=${this.rebuilder.electronVersion}`,
+        `--dist-url=${this.rebuilder.headerURL}`,
+      ];
+    }
+  }
+}
diff --git a/src/rebuild.ts b/src/rebuild.ts
@@ -28,6 +28,7 @@ export interface RebuildOptions {
   projectRootPath?: string;
   forceABI?: number;
   disablePreGypCopy?: boolean;
+  buildFromSource?: boolean;
 }
 
 export interface RebuilderOptions extends RebuildOptions {
@@ -60,6 +61,7 @@ export class Rebuilder implements IRebuilder {
   public msvsVersion?: string;
   public useElectronClang: boolean;
   public disablePreGypCopy: boolean;
+  public buildFromSource: boolean;
 
   constructor(options: RebuilderOptions) {
     this.lifecycle = options.lifecycle;
@@ -76,6 +78,7 @@ export class Rebuilder implements IRebuilder {
     this.prebuildTagPrefix = options.prebuildTagPrefix || 'v';
     this.msvsVersion = process.env.GYP_MSVS_VERSION;
     this.disablePreGypCopy = options.disablePreGypCopy || false;
+    this.buildFromSource = options.buildFromSource || false;
 
     if (this.useCache && this.force) {
       console.warn('[WARNING]: Electron Rebuild has force enabled and cache enabled, force take precedence and the cache will not be used.');

diff --git a/src/types.ts b/src/types.ts
@@ -23,6 +23,7 @@ export interface IRebuilder {
   msvsVersion?: string;
   platform: string;
   prebuildTagPrefix: string;
+  buildFromSource: boolean;
   useCache: boolean;
   useElectronClang: boolean;
 }
diff --git a/test/fixture/native-app1/package.json b/test/fixture/native-app1/package.json
@@ -22,7 +22,8 @@
     "farmhash": "3.2.1",
     "level": "6.0.0",
     "native-hello-world": "2.0.0",
-    "ref-napi": "1.4.2"
+    "ref-napi": "1.4.2",
+    "sqlite3": "5.1.6"
   },
   "optionalDependencies": {
     "bcrypt": "3.0.6"

diff --git a/test/helpers/module-setup.ts b/test/helpers/module-setup.ts
@@ -35,7 +35,7 @@ export async function resetTestModule(testModulePath: string, installModules = t
 }
 
 export async function cleanupTestModule(testModulePath: string): Promise<void> {
-  await fs.remove(testModulePath);
+  await fs.rmdir(testModulePath, { recursive: true, maxRetries: 10 });
   resetMSVSVersion();
 }
 

diff --git a/test/module-type-node-pre-gyp.ts b/test/module-type-node-pre-gyp.ts
@@ -0,0 +1,50 @@
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import { EventEmitter } from 'events';
+import path from 'path';
+
+import { cleanupTestModule, resetTestModule, TIMEOUT_IN_MILLISECONDS, TEST_MODULE_PATH as testModulePath } from './helpers/module-setup';
+import { NodePreGyp } from '../lib/module-type/node-pre-gyp';
+import { Rebuilder } from '../lib/rebuild';
+
+chai.use(chaiAsPromised);
+
+describe('node-pre-gyp', () => {
+  const modulePath = path.join(testModulePath, 'node_modules', 'sqlite3');
+  const rebuilderArgs = {
+    buildPath: testModulePath,
+    electronVersion: '8.0.0',
+    arch: process.arch,
+    lifecycle: new EventEmitter()
+  };
+
+  describe('Node-API support', function() {
+    this.timeout(TIMEOUT_IN_MILLISECONDS);
+
+    before(async () => await resetTestModule(testModulePath));
+    after(async () => await cleanupTestModule(testModulePath));
+
+    it('should find correct napi version and select napi args', async () => {
+      const rebuilder = new Rebuilder(rebuilderArgs);
+      const nodePreGyp = new NodePreGyp(rebuilder, modulePath);
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      expect(nodePreGyp.nodeAPI.getNapiVersion((await nodePreGyp.getSupportedNapiVersions())!)).to.equal(3);
+      expect(await nodePreGyp.getNodePreGypRuntimeArgs()).to.deep.equal([])
+    });
+
+    it('should not fail running node-pre-gyp', async () => {
+      const rebuilder = new Rebuilder(rebuilderArgs);
+      const nodePreGyp = new NodePreGyp(rebuilder, modulePath);
+      expect(await nodePreGyp.findPrebuiltModule()).to.equal(true);
+    });
+
+    it('should throw error with unsupported Electron version', async () => {
+      const rebuilder = new Rebuilder({
+        ...rebuilderArgs,
+        electronVersion: '2.0.0',
+      });
+      const nodePreGyp = new NodePreGyp(rebuilder, modulePath);
+      expect(nodePreGyp.findPrebuiltModule()).to.eventually.be.rejectedWith("Native module 'sqlite3' requires Node-API but Electron v2.0.0 does not support Node-API");
+    });
+  });
+});
diff --git a/test/module-type-prebuild-install.ts b/test/module-type-prebuild-install.ts
@@ -35,7 +35,10 @@ describe('prebuild-install', () => {
       ])
     });
 
-    it('should not fail running prebuild-install', async () => {
+    it('should not fail running prebuild-install', async function () {
+      if (process.platform === 'darwin' && process.arch === 'arm64') {
+        this.skip(); // farmhash module has no prebuilt binaries for ARM64
+      }
       const rebuilder = new Rebuilder(rebuilderArgs);
       const prebuildInstall = new PrebuildInstall(rebuilder, modulePath);
       expect(await prebuildInstall.findPrebuiltModule()).to.equal(true);

diff --git a/test/rebuild-napibuildversion.ts b/test/rebuild-napibuildversion.ts
@@ -26,7 +26,7 @@ describe('rebuild with napi_build_versions in binary config', async function ()
   // https://github.com/electron/rebuild/issues/554
   const archs = ['x64', 'arm64']
   for (const arch of archs) {
-    it(`${ arch } arch should have rebuilt bianry with 'napi_build_versions' array and 'libc' provided`, async () => {
+    it(`${ arch } arch should have rebuilt binary with 'napi_build_versions' array and 'libc' provided`, async () => {
       const libc = await detectLibc.family() || 'unknown'
       const binaryPath = napiBuildVersionSpecificPath(arch, libc)
 
@@ -38,7 +38,8 @@ describe('rebuild with napi_build_versions in binary config', async function ()
       await rebuild({
         buildPath: testModulePath,
         electronVersion: testElectronVersion,
-        arch
+        arch,
+        buildFromSource: true, // need to skip node-pre-gyp prebuilt binary
       });
 
       await expectNativeModuleToBeRebuilt(testModulePath, 'sqlite3');

diff --git a/test/rebuild.ts b/test/rebuild.ts
@@ -81,7 +81,7 @@ describe('rebuilder', () => {
         skipped++;
       });
       await rebuilder;
-      expect(skipped).to.equal(7);
+      expect(skipped).to.equal(8);
     });
 
     it('should rebuild all modules again when disabled but the electron ABI changed', async () => {

diff --git a/yarn.lock b/yarn.lock
@@ -3210,10 +3210,10 @@ nerf-dart@^1.0.0:
   resolved "https://registry.yarnpkg.com/nerf-dart/-/nerf-dart-1.0.0.tgz#e6dab7febf5ad816ea81cf5c629c5a0ebde72c1a"
   integrity sha512-EZSPZB70jiVsivaBLYDCyntd5eH8NTSMOn3rB+HxwdmKThGELLdYv8qVIMWvZEFy9w8ZZpW9h9OB32l1rGtj7g==
 
-node-abi@^3.0.0:
-  version "3.30.0"
-  resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.30.0.tgz#d84687ad5d24ca81cdfa912a36f2c5c19b137359"
-  integrity sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==
+node-abi@^3.45.0:
+  version "3.45.0"
+  resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.45.0.tgz#f568f163a3bfca5aacfce1fbeee1fa2cc98441f5"
+  integrity sha512-iwXuFrMAcFVi/ZoZiqq8BzAdsLw9kxDfTC0HMyjXfSL/6CSDAGD5UmR7azrAgWV1zKYq7dUUMj4owusBWKLsiQ==
   dependencies:
     semver "^7.3.5"