From cadf25c31285ffc97851cba6b1f518c997b648d1 Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Tue, 24 Sep 2024 12:22:18 -0700 Subject: [PATCH 1/7] fix: allow prebuilt binaries to be downloaded for target platform instead of hardcoded `process.platform`. Throws an error when attempting to cross-compile (`buildFromSource` || no prebuilds found) --- src/module-rebuilder.ts | 4 ++++ src/rebuild.ts | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/module-rebuilder.ts b/src/module-rebuilder.ts index 368ca249..f724d304 100644 --- a/src/module-rebuilder.ts +++ b/src/module-rebuilder.ts @@ -151,6 +151,10 @@ export class ModuleRebuilder { return true; } + if (this.rebuilder.platform !== process.platform) { + throw new Error("OS Limitation - It is not possible to cross-compile native modules from source") + } + return await this.rebuildNodeGypModule(cacheKey); } } diff --git a/src/rebuild.ts b/src/rebuild.ts index ff08b416..495cc334 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -19,6 +19,13 @@ export interface RebuildOptions { * The version of Electron to build against. */ electronVersion: string; + /** + * Override the target platform to something other than the host system platform. + * Note: This only applies to downloading prebuilt binaries. **It is not possible to cross-compile native modules** + * + * @defaultValue The system {@link https://nodejs.org/api/process.html#processplatform | `process.arch`} value + */ + platform?: string; /** * Override the target rebuild architecture to something other than the host system architecture. * @@ -132,7 +139,7 @@ export class Rebuilder implements IRebuilder { public lifecycle: EventEmitter; public buildPath: string; public electronVersion: string; - public platform: string = process.platform; + public platform: string public arch: string; public force: boolean; public headerURL: string; @@ -151,6 +158,7 @@ export class Rebuilder implements IRebuilder { this.lifecycle = options.lifecycle; this.buildPath = options.buildPath; this.electronVersion = options.electronVersion; + this.platform = options.platform || process.platform this.arch = options.arch || process.arch; this.force = options.force || false; this.headerURL = options.headerURL || 'https://www.electronjs.org/headers'; From 0a9a6f837869381288bfd871003f0c1a8e2b2618 Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Tue, 24 Sep 2024 13:03:05 -0700 Subject: [PATCH 2/7] Update src/rebuild.ts Co-authored-by: Erick Zhao --- src/rebuild.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebuild.ts b/src/rebuild.ts index 495cc334..144f9cf2 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -21,7 +21,7 @@ export interface RebuildOptions { electronVersion: string; /** * Override the target platform to something other than the host system platform. - * Note: This only applies to downloading prebuilt binaries. **It is not possible to cross-compile native modules** + * Note: This only applies to downloading prebuilt binaries. **It is not possible to cross-compile native modules.** * * @defaultValue The system {@link https://nodejs.org/api/process.html#processplatform | `process.arch`} value */ From fff22551593b4f41a51d8420164a4233d0b47c44 Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Tue, 24 Sep 2024 14:27:15 -0700 Subject: [PATCH 3/7] Added unit tests for each prebuild logic flow and node-gyp early exit/error. Added `platform` to `CacheOptions` cache key generation --- src/cache.ts | 2 ++ src/module-rebuilder.ts | 4 ---- src/module-type/node-gyp/node-gyp.ts | 4 ++++ src/rebuild.ts | 8 +++++--- src/types.ts | 2 +- test/module-type-node-gyp.ts | 20 ++++++++++++++++++++ test/module-type-node-pre-gyp.ts | 21 +++++++++++++++++++-- test/module-type-prebuild-install.ts | 24 ++++++++++++++++++++++-- test/module-type-prebuildify.ts | 20 ++++++++++++++++++++ 9 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/cache.ts b/src/cache.ts index 7b9fe403..e068245c 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -22,6 +22,7 @@ type HashTree = { [path: string]: string | HashTree }; type CacheOptions = { ABI: string; arch: string; + platform: string; debug: boolean; electronVersion: string; headerURL: string; @@ -159,6 +160,7 @@ export async function generateCacheKey(opts: CacheOptions): Promise { .update(path.basename(opts.modulePath)) .update(opts.ABI) .update(opts.arch) + .update(opts.platform) .update(opts.debug ? 'debug' : 'not debug') .update(opts.headerURL) .update(opts.electronVersion); diff --git a/src/module-rebuilder.ts b/src/module-rebuilder.ts index f724d304..368ca249 100644 --- a/src/module-rebuilder.ts +++ b/src/module-rebuilder.ts @@ -151,10 +151,6 @@ export class ModuleRebuilder { return true; } - if (this.rebuilder.platform !== process.platform) { - throw new Error("OS Limitation - It is not possible to cross-compile native modules from source") - } - return await this.rebuildNodeGypModule(cacheKey); } } diff --git a/src/module-type/node-gyp/node-gyp.ts b/src/module-type/node-gyp/node-gyp.ts index a7c60737..be2691a4 100644 --- a/src/module-type/node-gyp/node-gyp.ts +++ b/src/module-type/node-gyp/node-gyp.ts @@ -84,6 +84,10 @@ export class NodeGyp extends NativeModule { } async rebuildModule(): Promise { + if (this.rebuilder.platform !== process.platform) { + throw new Error("node-gyp does not support cross-compiling native modules from source") + } + if (this.modulePath.includes(' ')) { console.error('Attempting to build a module with a space in the path'); console.error('See https://github.com/nodejs/node-gyp/issues/65#issuecomment-368820565 for reasons why this may not work'); diff --git a/src/rebuild.ts b/src/rebuild.ts index 144f9cf2..845adb8e 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -23,9 +23,9 @@ export interface RebuildOptions { * Override the target platform to something other than the host system platform. * Note: This only applies to downloading prebuilt binaries. **It is not possible to cross-compile native modules.** * - * @defaultValue The system {@link https://nodejs.org/api/process.html#processplatform | `process.arch`} value + * @defaultValue The system {@link https://nodejs.org/api/process.html#processplatform | `process.platform`} value */ - platform?: string; + platform?: NodeJS.Platform; /** * Override the target rebuild architecture to something other than the host system architecture. * @@ -139,7 +139,7 @@ export class Rebuilder implements IRebuilder { public lifecycle: EventEmitter; public buildPath: string; public electronVersion: string; - public platform: string + public platform: NodeJS.Platform public arch: string; public force: boolean; public headerURL: string; @@ -207,6 +207,7 @@ export class Rebuilder implements IRebuilder { 'rebuilding with args:', this.buildPath, this.electronVersion, + this.platform, this.arch, extraModules, this.force, @@ -297,6 +298,7 @@ export class Rebuilder implements IRebuilder { cacheKey = await generateCacheKey({ ABI: this.ABI, arch: this.arch, + platform: this.platform, debug: this.debug, electronVersion: this.electronVersion, headerURL: this.headerURL, diff --git a/src/types.ts b/src/types.ts index 393fee3d..bf5eaf25 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,7 +21,7 @@ export interface IRebuilder { lifecycle: EventEmitter; mode: RebuildMode; msvsVersion?: string; - platform: string; + platform: NodeJS.Platform; prebuildTagPrefix: string; buildFromSource: boolean; useCache: boolean; diff --git a/test/module-type-node-gyp.ts b/test/module-type-node-gyp.ts index e9b4496e..d4801ff8 100644 --- a/test/module-type-node-gyp.ts +++ b/test/module-type-node-gyp.ts @@ -54,5 +54,25 @@ describe('node-gyp', () => { expect(args).to.not.include('--force-process-config'); }); }); + + context('cross-compilation', async () => { + it('throws error early', async () => { + const platform: NodeJS.Platform = 'win32' + const rebuilder = new Rebuilder({ + buildPath: testModulePath, + electronVersion: '15.3.0', + lifecycle: new EventEmitter(), + platform, + buildFromSource: true // to force node-gyp to execute + }); + const nodeGyp = new NodeGyp(rebuilder, testModulePath); + const executor = () => nodeGyp.rebuildModule(); + if (process.platform === platform) { + expect(executor).does.not.throw() + } else { + expect(executor).throws() + } + }) + }) }); }); diff --git a/test/module-type-node-pre-gyp.ts b/test/module-type-node-pre-gyp.ts index 4d0d9ce8..cf617e3e 100644 --- a/test/module-type-node-pre-gyp.ts +++ b/test/module-type-node-pre-gyp.ts @@ -5,7 +5,7 @@ import path from 'path'; import { cleanupTestModule, resetTestModule, TIMEOUT_IN_MILLISECONDS, TEST_MODULE_PATH as testModulePath } from './helpers/module-setup'; import { NodePreGyp } from '../lib/module-type/node-pre-gyp'; -import { Rebuilder } from '../lib/rebuild'; +import { Rebuilder, RebuilderOptions } from '../lib/rebuild'; chai.use(chaiAsPromised); @@ -13,7 +13,7 @@ describe('node-pre-gyp', function () { this.timeout(TIMEOUT_IN_MILLISECONDS); const modulePath = path.join(testModulePath, 'node_modules', 'sqlite3'); - const rebuilderArgs = { + const rebuilderArgs: RebuilderOptions = { buildPath: testModulePath, electronVersion: '8.0.0', arch: process.arch, @@ -64,4 +64,21 @@ describe('node-pre-gyp', function () { nodePreGyp = new NodePreGyp(rebuilder, modulePath); expect(await nodePreGyp.findPrebuiltModule()).to.equal(true); }); + + it('should download for target platform', async () => { + let rebuilder = new Rebuilder(rebuilderArgs); + let nodePreGyp = new NodePreGyp(rebuilder, modulePath); + expect(await nodePreGyp.findPrebuiltModule()).to.equal(true); + + let alternativePlatform: NodeJS.Platform; + if (process.platform === 'win32') { + alternativePlatform = 'darwin'; + } else { + alternativePlatform = 'win32' + } + + rebuilder = new Rebuilder({ ...rebuilderArgs, platform: alternativePlatform }); + nodePreGyp = new NodePreGyp(rebuilder, modulePath); + expect(await nodePreGyp.findPrebuiltModule()).to.equal(true); + }); }); diff --git a/test/module-type-prebuild-install.ts b/test/module-type-prebuild-install.ts index a9ed893b..f31f2697 100644 --- a/test/module-type-prebuild-install.ts +++ b/test/module-type-prebuild-install.ts @@ -5,13 +5,13 @@ import path from 'path'; import { cleanupTestModule, resetTestModule, TIMEOUT_IN_MILLISECONDS, TEST_MODULE_PATH as testModulePath } from './helpers/module-setup'; import { PrebuildInstall } from '../lib/module-type/prebuild-install'; -import { Rebuilder } from '../lib/rebuild'; +import { Rebuilder, RebuilderOptions } from '../lib/rebuild'; chai.use(chaiAsPromised); describe('prebuild-install', () => { const modulePath = path.join(testModulePath, 'node_modules', 'farmhash'); - const rebuilderArgs = { + const rebuilderArgs: RebuilderOptions = { buildPath: testModulePath, electronVersion: '8.0.0', arch: process.arch, @@ -52,5 +52,25 @@ describe('prebuild-install', () => { const prebuildInstall = new PrebuildInstall(rebuilder, modulePath); expect(prebuildInstall.findPrebuiltModule()).to.eventually.be.rejectedWith("Native module 'farmhash' requires Node-API but Electron v2.0.0 does not support Node-API"); }); + + it('should download for target platform', async function () { + if (process.platform === 'darwin' && process.arch === 'arm64') { + this.skip(); // farmhash module has no prebuilt binaries for ARM64 + } + let rebuilder = new Rebuilder(rebuilderArgs); + let prebuild = new PrebuildInstall(rebuilder, modulePath); + expect(await prebuild.findPrebuiltModule()).to.equal(true); + + let alternativePlatform: NodeJS.Platform; + if (process.platform === 'win32') { + alternativePlatform = 'darwin'; + } else { + alternativePlatform = 'win32' + } + + rebuilder = new Rebuilder({ ...rebuilderArgs, platform: alternativePlatform }); + prebuild = new PrebuildInstall(rebuilder, modulePath); + expect(await prebuild.findPrebuiltModule()).to.equal(true); + }); }); }); diff --git a/test/module-type-prebuildify.ts b/test/module-type-prebuildify.ts index d3579295..3325343d 100644 --- a/test/module-type-prebuildify.ts +++ b/test/module-type-prebuildify.ts @@ -113,4 +113,24 @@ describe('prebuildify', () => { }); }); }); + + describe('cross-platform downloads', async () => { + it('should download for target platform', async () => { + const fixtureDir = path.join(fixtureBaseDir, 'napi'); + let rebuilder = createRebuilder(); + let prebuildify = new Prebuildify(rebuilder, fixtureDir); + expect(await prebuildify.findPrebuiltModule()).to.equal(true); + + let alternativePlatform: NodeJS.Platform; + if (process.platform === 'win32') { + alternativePlatform = 'darwin'; + } else { + alternativePlatform = 'win32' + } + + rebuilder = createRebuilder({ platform: alternativePlatform }); + prebuildify = new Prebuildify(rebuilder, fixtureDir); + expect(await prebuildify.findPrebuiltModule()).to.equal(true); + }); + }) }); From 33de52a426cf55b0ef7a3e9e92d5bc1e6c5bebac Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Tue, 24 Sep 2024 21:46:30 -0700 Subject: [PATCH 4/7] test for thrown exception via boolean flag --- src/module-type/node-gyp/node-gyp.ts | 2 +- test/module-type-node-gyp.ts | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/module-type/node-gyp/node-gyp.ts b/src/module-type/node-gyp/node-gyp.ts index be2691a4..ed4f9be9 100644 --- a/src/module-type/node-gyp/node-gyp.ts +++ b/src/module-type/node-gyp/node-gyp.ts @@ -85,7 +85,7 @@ export class NodeGyp extends NativeModule { async rebuildModule(): Promise { if (this.rebuilder.platform !== process.platform) { - throw new Error("node-gyp does not support cross-compiling native modules from source") + throw new Error("node-gyp does not support cross-compiling native modules from source."); } if (this.modulePath.includes(' ')) { diff --git a/test/module-type-node-gyp.ts b/test/module-type-node-gyp.ts index d4801ff8..58a81edd 100644 --- a/test/module-type-node-gyp.ts +++ b/test/module-type-node-gyp.ts @@ -56,21 +56,29 @@ describe('node-gyp', () => { }); context('cross-compilation', async () => { - it('throws error early', async () => { + it('throws error early if platform mismatch', async () => { const platform: NodeJS.Platform = 'win32' const rebuilder = new Rebuilder({ buildPath: testModulePath, electronVersion: '15.3.0', lifecycle: new EventEmitter(), - platform, - buildFromSource: true // to force node-gyp to execute + platform }); const nodeGyp = new NodeGyp(rebuilder, testModulePath); - const executor = () => nodeGyp.rebuildModule(); + + const errorMessage = "node-gyp does not support cross-compiling native modules from source." + let errorThrown = false + const executor = () => nodeGyp.rebuildModule().catch((err) => { + if (err.message === errorMessage) { + errorThrown = true + } + }); + await executor() + if (process.platform === platform) { - expect(executor).does.not.throw() + expect(errorThrown).to.be.false } else { - expect(executor).throws() + expect(errorThrown).to.be.true } }) }) From 0fbaba08cc7055f394a3ad231ee630d2cbb6b663 Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Tue, 24 Sep 2024 22:16:02 -0700 Subject: [PATCH 5/7] skip testing rebuildModule when there isn't a platform mismatch, otherwise it kicks off full rebuild of modules and that timesout after 2000ms --- src/rebuild.ts | 2 +- test/module-type-node-gyp.ts | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/rebuild.ts b/src/rebuild.ts index 845adb8e..aca32ebb 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -139,7 +139,7 @@ export class Rebuilder implements IRebuilder { public lifecycle: EventEmitter; public buildPath: string; public electronVersion: string; - public platform: NodeJS.Platform + public platform: NodeJS.Platform; public arch: string; public force: boolean; public headerURL: string; diff --git a/test/module-type-node-gyp.ts b/test/module-type-node-gyp.ts index 58a81edd..64f001de 100644 --- a/test/module-type-node-gyp.ts +++ b/test/module-type-node-gyp.ts @@ -56,8 +56,14 @@ describe('node-gyp', () => { }); context('cross-compilation', async () => { - it('throws error early if platform mismatch', async () => { - const platform: NodeJS.Platform = 'win32' + it('throws error early if platform mismatch', async function () { + const platform: NodeJS.Platform = 'win32'; + + // we're verifying platform mismatch error throwing, not `rebuildModule` rebuilding. + if (process.platform === platform) { + this.skip(); + } + const rebuilder = new Rebuilder({ buildPath: testModulePath, electronVersion: '15.3.0', @@ -66,20 +72,14 @@ describe('node-gyp', () => { }); const nodeGyp = new NodeGyp(rebuilder, testModulePath); - const errorMessage = "node-gyp does not support cross-compiling native modules from source." - let errorThrown = false - const executor = () => nodeGyp.rebuildModule().catch((err) => { + const errorMessage = "node-gyp does not support cross-compiling native modules from source."; + let errorThrown = false; + await nodeGyp.rebuildModule().catch((err) => { if (err.message === errorMessage) { errorThrown = true } }); - await executor() - - if (process.platform === platform) { - expect(errorThrown).to.be.false - } else { - expect(errorThrown).to.be.true - } + expect(errorThrown).to.be.true; }) }) }); From 65cfff5cd31104086d892194996919563718736e Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Wed, 25 Sep 2024 07:41:45 -0700 Subject: [PATCH 6/7] use `chai-as-promised` for async throw/rejection testing --- test/module-type-node-gyp.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/module-type-node-gyp.ts b/test/module-type-node-gyp.ts index 64f001de..80d3bad5 100644 --- a/test/module-type-node-gyp.ts +++ b/test/module-type-node-gyp.ts @@ -1,10 +1,13 @@ import { EventEmitter } from 'events'; -import { expect } from 'chai'; +import chai, { expect } from 'chai'; +import chaiAsPromised from 'chai-as-promised'; import { cleanupTestModule, resetTestModule, TEST_MODULE_PATH as testModulePath } from './helpers/module-setup'; import { NodeGyp } from '../lib/module-type/node-gyp/node-gyp'; import { Rebuilder } from '../lib/rebuild'; +chai.use(chaiAsPromised); + describe('node-gyp', () => { describe('buildArgs', () => { @@ -73,13 +76,7 @@ describe('node-gyp', () => { const nodeGyp = new NodeGyp(rebuilder, testModulePath); const errorMessage = "node-gyp does not support cross-compiling native modules from source."; - let errorThrown = false; - await nodeGyp.rebuildModule().catch((err) => { - if (err.message === errorMessage) { - errorThrown = true - } - }); - expect(errorThrown).to.be.true; + expect(nodeGyp.rebuildModule()).to.eventually.be.rejectedWith(new Error(errorMessage)) }) }) }); From 79d73e05b0c8ffc328b26eaae07d6bee3b4225fc Mon Sep 17 00:00:00 2001 From: Mike Maietta Date: Wed, 25 Sep 2024 10:58:20 -0700 Subject: [PATCH 7/7] semicolons --- src/rebuild.ts | 2 +- test/module-type-node-gyp.ts | 4 ++-- test/module-type-node-pre-gyp.ts | 2 +- test/module-type-prebuild-install.ts | 2 +- test/module-type-prebuildify.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rebuild.ts b/src/rebuild.ts index aca32ebb..a075ab07 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -158,7 +158,7 @@ export class Rebuilder implements IRebuilder { this.lifecycle = options.lifecycle; this.buildPath = options.buildPath; this.electronVersion = options.electronVersion; - this.platform = options.platform || process.platform + this.platform = options.platform || process.platform; this.arch = options.arch || process.arch; this.force = options.force || false; this.headerURL = options.headerURL || 'https://www.electronjs.org/headers'; diff --git a/test/module-type-node-gyp.ts b/test/module-type-node-gyp.ts index 80d3bad5..e748e6b4 100644 --- a/test/module-type-node-gyp.ts +++ b/test/module-type-node-gyp.ts @@ -60,11 +60,11 @@ describe('node-gyp', () => { context('cross-compilation', async () => { it('throws error early if platform mismatch', async function () { - const platform: NodeJS.Platform = 'win32'; + let platform: NodeJS.Platform = 'darwin'; // we're verifying platform mismatch error throwing, not `rebuildModule` rebuilding. if (process.platform === platform) { - this.skip(); + platform = 'win32'; } const rebuilder = new Rebuilder({ diff --git a/test/module-type-node-pre-gyp.ts b/test/module-type-node-pre-gyp.ts index cf617e3e..8d3c509c 100644 --- a/test/module-type-node-pre-gyp.ts +++ b/test/module-type-node-pre-gyp.ts @@ -74,7 +74,7 @@ describe('node-pre-gyp', function () { if (process.platform === 'win32') { alternativePlatform = 'darwin'; } else { - alternativePlatform = 'win32' + alternativePlatform = 'win32'; } rebuilder = new Rebuilder({ ...rebuilderArgs, platform: alternativePlatform }); diff --git a/test/module-type-prebuild-install.ts b/test/module-type-prebuild-install.ts index f31f2697..a5f0756e 100644 --- a/test/module-type-prebuild-install.ts +++ b/test/module-type-prebuild-install.ts @@ -65,7 +65,7 @@ describe('prebuild-install', () => { if (process.platform === 'win32') { alternativePlatform = 'darwin'; } else { - alternativePlatform = 'win32' + alternativePlatform = 'win32'; } rebuilder = new Rebuilder({ ...rebuilderArgs, platform: alternativePlatform }); diff --git a/test/module-type-prebuildify.ts b/test/module-type-prebuildify.ts index 3325343d..1016b28b 100644 --- a/test/module-type-prebuildify.ts +++ b/test/module-type-prebuildify.ts @@ -125,7 +125,7 @@ describe('prebuildify', () => { if (process.platform === 'win32') { alternativePlatform = 'darwin'; } else { - alternativePlatform = 'win32' + alternativePlatform = 'win32'; } rebuilder = createRebuilder({ platform: alternativePlatform });