diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d99a187e7c..e67898b667 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -52,12 +52,12 @@ jobs: os: [ubuntu-24.04, macos-14, windows-2022] node-version: ['22'] # Must include the minimum deno version from the `DENO_VERSION_RANGE` constant in `node/bridge.ts`. - deno-version: ['v1.39.0', 'v1.46.3'] + deno-version: ['v2.1.4'] include: - os: ubuntu-24.04 # Earliest supported version node-version: '14.16.0' - deno-version: 'v1.46.3' + deno-version: 'v2.1.4' fail-fast: false steps: # Increasing the maximum number of open files. See: @@ -170,7 +170,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.46.3 + deno-version: v2.1.4 if: ${{ !steps.release-check.outputs.IS_RELEASE }} - name: Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 diff --git a/package-lock.json b/package-lock.json index e8e8f2cf87..a38764da4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1769,6 +1769,25 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@isaacs/fs-minipass/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/@isaacs/string-locale-compare": { "version": "1.1.0", "dev": true, @@ -11840,7 +11859,8 @@ }, "node_modules/common-path-prefix": { "version": "3.0.0", - "license": "ISC" + "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", + "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==" }, "node_modules/commondir": { "version": "1.0.1", @@ -21668,6 +21688,11 @@ "node": ">=8" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" + }, "node_modules/pacote": { "version": "13.6.2", "dev": true, @@ -21840,15 +21865,15 @@ "license": "MIT" }, "node_modules/path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": ">=16 || 14 >=14.18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -26799,13 +26824,14 @@ "p-wait-for": "^4.1.0", "path-key": "^4.0.0", "semver": "^7.3.8", + "tar": "^7.4.3", "tmp-promise": "^3.0.3", "urlpattern-polyfill": "8.0.2", "uuid": "^9.0.0" }, "devDependencies": { "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^14.18.32", + "@types/node": "^16.18.122", "@types/semver": "^7.3.9", "@types/uuid": "^9.0.0", "@vitest/coverage-v8": "^0.34.0", @@ -26814,7 +26840,6 @@ "cpy": "^9.0.1", "cross-env": "^7.0.3", "nock": "^13.2.4", - "tar": "^6.1.11", "typescript": "^5.0.0", "vitest": "^0.34.0" }, @@ -26822,6 +26847,12 @@ "node": "^14.16.0 || >=16.0.0" } }, + "packages/edge-bundler/node_modules/@types/node": { + "version": "16.18.122", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.122.tgz", + "integrity": "sha512-rF6rUBS80n4oK16EW8nE75U+9fw0SSUgoPtWSvHhPXdT7itbvmS7UjB/jyM8i3AkvI6yeSM5qCwo+xN0npGDHg==", + "dev": true + }, "packages/edge-bundler/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -26907,6 +26938,14 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "packages/edge-bundler/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "engines": { + "node": ">=18" + } + }, "packages/edge-bundler/node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -26969,6 +27008,32 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "packages/edge-bundler/node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "packages/edge-bundler/node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "packages/edge-bundler/node_modules/human-signals": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-3.0.1.tgz", @@ -26994,6 +27059,54 @@ "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, + "packages/edge-bundler/node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "packages/edge-bundler/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "packages/edge-bundler/node_modules/minizlib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "dependencies": { + "minipass": "^7.0.4", + "rimraf": "^5.0.5" + }, + "engines": { + "node": ">= 18" + } + }, + "packages/edge-bundler/node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "packages/edge-bundler/node_modules/npm-run-path": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", @@ -27022,6 +27135,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/edge-bundler/node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "packages/edge-bundler/node_modules/rimraf/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "packages/edge-bundler/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -27060,6 +27206,22 @@ "node": ">=8" } }, + "packages/edge-bundler/node_modules/tar": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "packages/edge-bundler/node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -27076,6 +27238,14 @@ "node": ">=6" } }, + "packages/edge-bundler/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } + }, "packages/edge-bundler/node_modules/zip-stream": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", diff --git a/packages/edge-bundler/deno/lib/common.ts b/packages/edge-bundler/deno/lib/common.ts index a3f6ced471..104bedabe5 100644 --- a/packages/edge-bundler/deno/lib/common.ts +++ b/packages/edge-bundler/deno/lib/common.ts @@ -42,7 +42,7 @@ const loadWithRetry = (specifier: string, delay = 1000, maxTry = 3) => { maxTry, }); } catch (error) { - if (isTooManyTries(error)) { + if (error instanceof Error && isTooManyTries(error)) { console.error(`Loading ${specifier} failed after ${maxTry} tries.`); } throw error; diff --git a/packages/edge-bundler/node/bridge.ts b/packages/edge-bundler/node/bridge.ts index a597bd3275..7b435fc0ad 100644 --- a/packages/edge-bundler/node/bridge.ts +++ b/packages/edge-bundler/node/bridge.ts @@ -12,11 +12,7 @@ import { getLogger, Logger } from './logger.js' import { getBinaryExtension } from './platform.js' const DENO_VERSION_FILE = 'version.txt' - -// When updating DENO_VERSION_RANGE, ensure that the deno version -// on the netlify/buildbot build image satisfies this range! -// https://github.com/netlify/buildbot/blob/f9c03c9dcb091d6570e9d0778381560d469e78ad/build-image/noble/Dockerfile#L410 -const DENO_VERSION_RANGE = '1.39.0 - 1.46.3' +const DENO_VERSION_RANGE = '^2.1.4' type OnBeforeDownloadHook = () => void | Promise type OnAfterDownloadHook = (error?: Error) => void | Promise @@ -37,6 +33,7 @@ interface ProcessRef { } interface RunOptions { + cwd?: string env?: NodeJS.ProcessEnv extendEnv?: boolean pipeOutput?: boolean @@ -241,11 +238,11 @@ class DenoBridge { // process, awaiting its execution. async run( args: string[], - { env: inputEnv, extendEnv = true, rejectOnExitCode = true, stderr, stdout }: RunOptions = {}, + { cwd, env: inputEnv, extendEnv = true, rejectOnExitCode = true, stderr, stdout }: RunOptions = {}, ) { const { path: binaryPath } = await this.getBinaryPath() const env = this.getEnvironmentVariables(inputEnv) - const options: Options = { env, extendEnv, reject: rejectOnExitCode } + const options: Options = { cwd, env, extendEnv, reject: rejectOnExitCode } return DenoBridge.runWithBinary(binaryPath, args, { options, stderr, stdout }) } diff --git a/packages/edge-bundler/node/bundle.ts b/packages/edge-bundler/node/bundle.ts index 522130a704..9f460bebd5 100644 --- a/packages/edge-bundler/node/bundle.ts +++ b/packages/edge-bundler/node/bundle.ts @@ -1,6 +1,7 @@ export enum BundleFormat { ESZIP2 = 'eszip2', JS = 'js', + TARBALL = 'tar', } export interface Bundle { diff --git a/packages/edge-bundler/node/bundler.test.ts b/packages/edge-bundler/node/bundler.test.ts index b1e23a57fa..96855b522e 100644 --- a/packages/edge-bundler/node/bundler.test.ts +++ b/packages/edge-bundler/node/bundler.test.ts @@ -5,10 +5,10 @@ import process from 'process' import { pathToFileURL } from 'url' import tmp from 'tmp-promise' -import { test, expect, vi } from 'vitest' +import { test, expect, vi, describe } from 'vitest' import { importMapSpecifier } from '../shared/consts.js' -import { runESZIP, useFixture } from '../test/util.js' +import { runESZIP, runTarball, useFixture } from '../test/util.js' import { BundleError } from './bundle_error.js' import { bundle, BundleOptions } from './bundler.js' @@ -105,15 +105,9 @@ test('Adds a custom error property to user errors during bundling', async () => } catch (error) { expect(error).toBeInstanceOf(BundleError) const [messageBeforeStack] = (error as BundleError).message.split('at (file://') - expect(messageBeforeStack).toMatchInlineSnapshot(` - "error: Uncaught (in promise) Error: The module's source code could not be parsed: Unexpected eof at file:///root/functions/func1.ts:1:27 - - export default async () => - ~ - const ret = new Error(getStringFromWasm0(arg0, arg1)); - ^ - " - `) + expect(messageBeforeStack).toContain( + `The module's source code could not be parsed: Unexpected eof at file:///root/functions/func1.ts:1:27`, + ) expect((error as BundleError).customErrorInfo).toEqual({ location: { format: 'eszip', @@ -502,7 +496,7 @@ test('Loads npm modules from bare specifiers', async () => { const { func1 } = await runESZIP(bundlePath, vendorDirectory.path) expect(func1).toBe( - `JavaScript, APIs${process.cwd()}, Markup${process.cwd()}`, + `JavaScript, APIs${process.platform}, Markup${process.platform}, TmV0bGlmeQ==`, ) await cleanup() @@ -641,3 +635,48 @@ test('Loads edge functions from the Frameworks API', async () => { await cleanup() }) + +describe('Produces a tarball bundle', () => { + test('Using npm modules', async () => { + const systemLogger = vi.fn() + const { basePath, cleanup, distPath } = await useFixture('imports_npm_module', { copyDirectory: true }) + const sourceDirectory = join(basePath, 'functions') + const declarations: Declaration[] = [ + { + function: 'func1', + path: '/func1', + }, + ] + const vendorDirectory = await tmp.dir() + + await bundle([sourceDirectory], distPath, declarations, { + basePath, + featureFlags: { + edge_bundler_generate_tarball: true, + }, + importMapPaths: [join(basePath, 'import_map.json')], + vendorDirectory: vendorDirectory.path, + systemLogger, + }) + + expect( + systemLogger.mock.calls.find((call) => call[0] === 'Could not track dependencies in edge function:'), + ).toBeUndefined() + + const expectedOutput = `JavaScript, APIs${process.platform}, Markup${process.platform}, TmV0bGlmeQ==` + + const manifestFile = await readFile(resolve(distPath, 'manifest.json'), 'utf8') + const manifest = JSON.parse(manifestFile) + + const tarballPath = join(distPath, manifest.bundles[0].asset) + const tarballResult = await runTarball(tarballPath) + expect(tarballResult.func1).toBe(expectedOutput) + + const eszipPath = join(distPath, manifest.bundles[1].asset) + const eszipResult = await runESZIP(eszipPath, vendorDirectory.path) + expect(eszipResult.func1).toBe(expectedOutput) + + await cleanup() + await rm(vendorDirectory.path, { force: true, recursive: true }) + }) +}, 10_000) diff --git a/packages/edge-bundler/node/bundler.ts b/packages/edge-bundler/node/bundler.ts index 4c001a4e62..0548a0676d 100644 --- a/packages/edge-bundler/node/bundler.ts +++ b/packages/edge-bundler/node/bundler.ts @@ -15,6 +15,7 @@ import { EdgeFunction } from './edge_function.js' import { FeatureFlags, getFlags } from './feature_flags.js' import { findFunctions } from './finder.js' import { bundle as bundleESZIP } from './formats/eszip.js' +import { bundle as bundleTarball } from './formats/tarball.js' import { ImportMap } from './import_map.js' import { getLogger, LogFunction, Logger } from './logger.js' import { writeManifest } from './manifest.js' @@ -114,27 +115,47 @@ export const bundle = async ( vendorDirectory, }) + const bundles: Bundle[] = [] + + if (featureFlags.edge_bundler_generate_tarball) { + bundles.push( + await bundleTarball({ + basePath, + buildID, + debug, + deno, + distDirectory, + functions, + featureFlags, + importMap: importMap.clone(), + vendorDirectory: vendor?.directory, + }), + ) + } + if (vendor) { importMap.add(vendor.importMap) } - const functionBundle = await bundleESZIP({ - basePath, - buildID, - debug, - deno, - distDirectory, - externals, - functions, - featureFlags, - importMap, - vendorDirectory: vendor?.directory, - }) + bundles.push( + await bundleESZIP({ + basePath, + buildID, + debug, + deno, + distDirectory, + externals, + functions, + featureFlags, + importMap, + vendorDirectory: vendor?.directory, + }), + ) // The final file name of the bundles contains a SHA256 hash of the contents, // which we can only compute now that the files have been generated. So let's // rename the bundles to their permanent names. - await createFinalBundles([functionBundle], distDirectory, buildID) + await createFinalBundles(bundles, distDirectory, buildID) // Retrieving a configuration object for each function. // Run `getFunctionConfig` in parallel as it is a non-trivial operation and spins up deno @@ -165,7 +186,7 @@ export const bundle = async ( }) const manifest = await writeManifest({ - bundles: [functionBundle], + bundles, declarations, distDirectory, featureFlags, diff --git a/packages/edge-bundler/node/config.ts b/packages/edge-bundler/node/config.ts index 40b865b299..02164baff5 100644 --- a/packages/edge-bundler/node/config.ts +++ b/packages/edge-bundler/node/config.ts @@ -99,12 +99,14 @@ export const getFunctionConfig = async ({ [ 'run', '--allow-env', + '--allow-import', '--allow-net', '--allow-read', `--allow-write=${collector.path}`, '--quiet', `--import-map=${importMap.toDataURL()}`, '--no-config', + '--no-lock', '--node-modules-dir=false', extractorPath, pathToFileURL(func.path).href, diff --git a/packages/edge-bundler/node/feature_flags.ts b/packages/edge-bundler/node/feature_flags.ts index bbb1af7173..eb27ce1528 100644 --- a/packages/edge-bundler/node/feature_flags.ts +++ b/packages/edge-bundler/node/feature_flags.ts @@ -1,4 +1,6 @@ -const defaultFlags = {} +const defaultFlags = { + edge_bundler_generate_tarball: false, +} type FeatureFlag = keyof typeof defaultFlags type FeatureFlags = Partial> diff --git a/packages/edge-bundler/node/formats/eszip.ts b/packages/edge-bundler/node/formats/eszip.ts index 432c07ff36..09df0229de 100644 --- a/packages/edge-bundler/node/formats/eszip.ts +++ b/packages/edge-bundler/node/formats/eszip.ts @@ -57,7 +57,7 @@ const bundleESZIP = async ({ importMapData, vendorDirectory, } - const flags = ['--allow-all', '--no-config', `--import-map=${bundlerImportMap}`] + const flags = ['--allow-all', '--no-config', '--no-lock', `--import-map=${bundlerImportMap}`] if (!debug) { flags.push('--quiet') diff --git a/packages/edge-bundler/node/formats/tarball.ts b/packages/edge-bundler/node/formats/tarball.ts new file mode 100644 index 0000000000..811bd6e862 --- /dev/null +++ b/packages/edge-bundler/node/formats/tarball.ts @@ -0,0 +1,239 @@ +import { promises as fs } from 'fs' +import path from 'path' +import { fileURLToPath, pathToFileURL } from 'url' + +import { resolve as importMapResolve } from '@import-maps/resolve' +import { nodeFileTrace, resolve as nftResolve } from '@vercel/nft' +import commonPathPrefix from 'common-path-prefix' +import * as tar from 'tar' +import tmp from 'tmp-promise' + +import { DenoBridge } from '../bridge.js' +import { Bundle, BundleFormat } from '../bundle.js' +import { EdgeFunction } from '../edge_function.js' +import { FeatureFlags } from '../feature_flags.js' +import { ImportMap } from '../import_map.js' +import { getStringHash, readFileAndHash } from '../utils/sha256.js' +import { transpile, TYPESCRIPT_EXTENSIONS } from '../utils/typescript.js' +import { ModuleGraphJson } from '../vendor/module_graph/module_graph.js' + +const TARBALL_EXTENSION = '.tar' +const TARBALL_SRC_DIRECTORY = 'src' + +interface Manifest { + functions: Record + version: number +} + +interface DenoInfoOptions { + basePath: string + deno: DenoBridge + denoDir: string + entrypoints: string[] + importMap: ImportMap +} + +const getDenoInfo = async ({ basePath, deno, denoDir, entrypoints, importMap }: DenoInfoOptions) => { + const { stdout } = await deno.run( + ['info', '--no-lock', '--import-map', importMap.toDataURL(), '--json', ...entrypoints], + { + cwd: basePath, + env: { + DENO_DIR: denoDir, + }, + }, + ) + + return JSON.parse(stdout) as ModuleGraphJson +} + +interface BundleTarballOptions { + basePath: string + buildID: string + debug?: boolean + deno: DenoBridge + distDirectory: string + featureFlags: FeatureFlags + functions: EdgeFunction[] + importMap: ImportMap + vendorDirectory?: string +} + +const resolveHTTPSSpecifier = (moduleGraph: ModuleGraphJson, specifier: string) => { + for (const mod of moduleGraph.modules) { + if (mod.specifier === specifier && mod.local) { + return { + localPath: mod.local, + isTypeScript: TYPESCRIPT_EXTENSIONS.has(path.extname(specifier)), + } + } + } +} + +const getUnixPath = (input: string) => input.split(path.sep).join('/') + +export const bundle = async ({ + basePath, + buildID, + deno, + distDirectory, + functions, + importMap, + vendorDirectory, +}: BundleTarballOptions): Promise => { + const hashes = new Map() + const sideFilesDir = await tmp.dir({ unsafeCleanup: true }) + const cleanup = [sideFilesDir.cleanup] + + let denoDir = vendorDirectory ? path.join(vendorDirectory, 'deno_dir') : undefined + + if (!denoDir) { + const tmpDir = await tmp.dir({ unsafeCleanup: true }) + + denoDir = tmpDir.path + + cleanup.push(tmpDir.cleanup) + } + + const manifest: Manifest = { + functions: {}, + version: 1, + } + const entrypoints: string[] = [] + + for (const func of functions) { + entrypoints.push(func.path) + + manifest.functions[func.name] = getUnixPath(path.relative(basePath, func.path)) + } + + const manifestPath = path.join(sideFilesDir.path, 'manifest.json') + const manifestContents = JSON.stringify(manifest) + hashes.set('manifest', getStringHash(manifestContents)) + await fs.writeFile(manifestPath, manifestContents) + + const denoConfigPath = path.join(sideFilesDir.path, 'deno.json') + const denoConfigContents = JSON.stringify(importMap.getContentsWithRelativePaths()) + hashes.set('config', getStringHash(denoConfigContents)) + await fs.writeFile(denoConfigPath, denoConfigContents) + + const tsPaths = new Set() + const rootDirectory = commonPathPrefix([basePath, denoDir]) + const moduleGraph = await getDenoInfo({ + basePath, + deno, + denoDir, + entrypoints, + importMap, + }) + + const baseURL = pathToFileURL(basePath) + const { fileList } = await nodeFileTrace(entrypoints, { + base: rootDirectory, + processCwd: basePath, + readFile: async (filePath: string) => { + if (TYPESCRIPT_EXTENSIONS.has(path.extname(filePath)) || tsPaths.has(filePath)) { + const transpiled = await transpile(filePath) + + hashes.set(filePath, getStringHash(transpiled)) + + return transpiled + } + + const { contents, hash } = await readFileAndHash(filePath) + + hashes.set(filePath, hash) + + return contents + }, + resolve: async (initialSpecifier, ...args) => { + let specifier = initialSpecifier + + // Start by checking whether the specifier matches any import map defined + // by the user. + const { matched, resolvedImport } = importMapResolve( + initialSpecifier, + importMap.getContentsWithURLObjects(), + baseURL, + ) + + // If it does, the resolved import is the specifier we'll evaluate going + // forward. + if (matched && resolvedImport.protocol === 'file:') { + specifier = fileURLToPath(resolvedImport).replace(/\\/g, '/') + } + + // If the specifier is an HTTPS import, we need to resolve it to a local + // file first. + if (specifier.startsWith('https://')) { + const resolved = resolveHTTPSSpecifier(moduleGraph, specifier) + + if (resolved) { + if (resolved.isTypeScript) { + tsPaths.add(resolved.localPath) + } + + specifier = resolved.localPath + } + } + + return nftResolve(specifier, ...args) + }, + }) + + // Computing a stable hash of the file list. + const hash = getStringHash( + [...hashes.keys()] + .sort() + .map((path) => hashes.get(path)) + .filter(Boolean) + .join(','), + ) + + const absolutePaths = [...fileList].map((relativePath) => path.resolve(rootDirectory, relativePath)) + const tarballPath = path.join(distDirectory, buildID + TARBALL_EXTENSION) + + await fs.mkdir(path.dirname(tarballPath), { recursive: true }) + + await tar.create( + { + cwd: rootDirectory, + file: tarballPath, + preservePaths: true, + onWriteEntry(entry) { + if (entry.path === denoConfigPath) { + entry.path = `./${TARBALL_SRC_DIRECTORY}/deno.json` + + return + } + + if (entry.path === manifestPath) { + entry.path = `./${TARBALL_SRC_DIRECTORY}/___netlify-edge-functions.json` + + return + } + + if (entry.path.startsWith(denoDir)) { + const tarPath = getUnixPath(path.relative(denoDir, entry.path)) + + entry.path = `./deno_dir/${tarPath}` + + return + } + + const tarPath = getUnixPath(path.relative(basePath, entry.path)) + + entry.path = `./${TARBALL_SRC_DIRECTORY}/${tarPath}` + }, + }, + [...absolutePaths, manifestPath, denoConfigPath], + ) + + await Promise.all(cleanup) + + return { + extension: TARBALL_EXTENSION, + format: BundleFormat.TARBALL, + hash, + } +} diff --git a/packages/edge-bundler/node/import_map.ts b/packages/edge-bundler/node/import_map.ts index c3cabfe75d..d31edd7247 100644 --- a/packages/edge-bundler/node/import_map.ts +++ b/packages/edge-bundler/node/import_map.ts @@ -200,6 +200,29 @@ export class ImportMap { } } + getContentsWithRelativePaths() { + let imports: Imports = {} + let scopes: Record = {} + + this.sources.forEach((file) => { + imports = { ...imports, ...file.imports } + scopes = { ...scopes, ...file.scopes } + }) + + // Internal imports must come last, because we need to guarantee that + // `netlify:edge` isn't user-defined. + Object.entries(INTERNAL_IMPORTS).forEach((internalImport) => { + const [specifier, url] = internalImport + + imports[specifier] = url + }) + + return { + imports, + scopes, + } + } + // The same as `getContents`, but the URLs are represented as URL objects // instead of strings. This is compatible with the `ParsedImportMap` type // from the `@import-maps/resolve` library. diff --git a/packages/edge-bundler/node/npm_dependencies.ts b/packages/edge-bundler/node/npm_dependencies.ts index f3e4aa6cbc..a1152b90fc 100644 --- a/packages/edge-bundler/node/npm_dependencies.ts +++ b/packages/edge-bundler/node/npm_dependencies.ts @@ -13,8 +13,7 @@ import tmp from 'tmp-promise' import { ImportMap } from './import_map.js' import { Logger } from './logger.js' import { pathsBetween } from './utils/fs.js' - -const TYPESCRIPT_EXTENSIONS = new Set(['.ts', '.tsx', '.cts', '.ctsx', '.mts', '.mtsx']) +import { transpile, TYPESCRIPT_EXTENSIONS } from './utils/typescript.js' const slugifyPackageName = (specifier: string) => { if (!specifier.startsWith('@')) return specifier @@ -118,15 +117,7 @@ const getNPMSpecifiers = async ({ basePath, functions, importMap, environment, r // If this is a TypeScript file, we need to compile in before we can // parse it. if (TYPESCRIPT_EXTENSIONS.has(path.extname(filePath))) { - const compiled = await build({ - bundle: false, - entryPoints: [filePath], - logLevel: 'silent', - platform: 'node', - write: false, - }) - - return compiled.outputFiles[0].text + return transpile(filePath) } return fs.readFile(filePath, 'utf8') diff --git a/packages/edge-bundler/node/server/server.ts b/packages/edge-bundler/node/server/server.ts index cd2c8af690..bfa2ddedca 100644 --- a/packages/edge-bundler/node/server/server.ts +++ b/packages/edge-bundler/node/server/server.ts @@ -125,7 +125,7 @@ const prepareServer = ({ // the `stage2Path` file as well as all of their dependencies. // Consumers such as the CLI can use this information to watch all the // relevant files and issue an isolate restart when one of them changes. - const { stdout } = await deno.run(['info', '--json', stage2Path]) + const { stdout } = await deno.run(['info', '--no-lock', '--json', stage2Path]) graph = JSON.parse(stdout) } catch { @@ -320,7 +320,7 @@ export const serve = async ({ // Downloading latest types if needed. await ensureLatestTypes(deno, logger) - const flags = ['--allow-all', '--no-config'] + const flags = ['--allow-all', '--no-config', '--no-lock'] if (certificatePath) { flags.push(`--cert=${certificatePath}`) diff --git a/packages/edge-bundler/node/utils/sha256.ts b/packages/edge-bundler/node/utils/sha256.ts index 03c835e64d..135baee5c0 100644 --- a/packages/edge-bundler/node/utils/sha256.ts +++ b/packages/edge-bundler/node/utils/sha256.ts @@ -1,7 +1,8 @@ -import crypto from 'crypto' -import fs from 'fs' +import { Buffer } from 'node:buffer' +import crypto from 'node:crypto' +import fs from 'node:fs' -const getFileHash = (path: string): Promise => { +export const getFileHash = (path: string): Promise => { const hash = crypto.createHash('sha256') hash.setEncoding('hex') @@ -20,4 +21,37 @@ const getFileHash = (path: string): Promise => { }) } -export { getFileHash } +export const getStringHash = (input: string) => { + const hash = crypto.createHash('sha256') + + hash.setEncoding('hex') + hash.update(input) + + return hash.digest('hex') +} + +export const readFileAndHash = (path: string) => { + const file = fs.createReadStream(path) + const hash = crypto.createHash('sha256') + const chunks: Uint8Array[] = [] + + hash.setEncoding('hex') + + return new Promise<{ contents: string; hash: string }>((resolve, reject) => { + file + .on('data', (chunk) => { + chunks.push(Buffer.from(chunk)) + + hash.update(chunk.toString()) + }) + .on('error', reject) + .on('end', () => { + const contents = Buffer.concat(chunks).toString('utf8') + + return resolve({ + contents, + hash: hash.digest('hex'), + }) + }) + }) +} diff --git a/packages/edge-bundler/node/utils/typescript.ts b/packages/edge-bundler/node/utils/typescript.ts new file mode 100644 index 0000000000..c0ffe8e2b2 --- /dev/null +++ b/packages/edge-bundler/node/utils/typescript.ts @@ -0,0 +1,22 @@ +import { build } from 'esbuild' + +export const TYPESCRIPT_EXTENSIONS = new Set(['.ts', '.tsx', '.cts', '.ctsx', '.mts', '.mtsx']) + +export const transpile = async (filePath: string) => { + const result = await build({ + bundle: false, + entryPoints: [filePath], + loader: { + // esbuild uses the extension of each entrypoint to determine the + // right loader to use. This doesn't work with the internal files + // from the Deno cache, so we must tell it to use the TypeScript + // loader for any files without an extension. + '': 'ts', + }, + logLevel: 'silent', + platform: 'node', + write: false, + }) + + return result.outputFiles[0].text +} diff --git a/packages/edge-bundler/package.json b/packages/edge-bundler/package.json index 7354a048c6..80959c95e3 100644 --- a/packages/edge-bundler/package.json +++ b/packages/edge-bundler/package.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^14.18.32", + "@types/node": "^16.18.122", "@types/semver": "^7.3.9", "@types/uuid": "^9.0.0", "@vitest/coverage-v8": "^0.34.0", @@ -52,7 +52,6 @@ "cpy": "^9.0.1", "cross-env": "^7.0.3", "nock": "^13.2.4", - "tar": "^6.1.11", "typescript": "^5.0.0", "vitest": "^0.34.0" }, @@ -80,6 +79,7 @@ "p-wait-for": "^4.1.0", "path-key": "^4.0.0", "semver": "^7.3.8", + "tar": "^7.4.3", "tmp-promise": "^3.0.3", "urlpattern-polyfill": "8.0.2", "uuid": "^9.0.0" diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts index cb73fdbf0d..e673b6dea1 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts @@ -1,14 +1,10 @@ import parent1 from 'parent-1' -import parent3 from './lib/util.ts' +import parent3 from './helpers/util.ts' import { echo, parent2 } from 'alias:helper' -import { HTMLRewriter } from 'html-rewriter' - -await Promise.resolve() - -new HTMLRewriter() +import { encode as base64Encode } from "https://deno.land/std@0.194.0/encoding/base64.ts"; export default async () => { - const text = [parent1('JavaScript'), parent2('APIs'), parent3('Markup')].join(', ') + const text = [parent1('JavaScript'), parent2('APIs'), parent3('Markup'), base64Encode("Netlify")].join(', ') return new Response(echo(text)) } diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/lib/util.ts b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/helpers/util.ts similarity index 100% rename from packages/edge-bundler/test/fixtures/imports_npm_module/functions/lib/util.ts rename to packages/edge-bundler/test/fixtures/imports_npm_module/functions/helpers/util.ts diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json b/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json index 5ef2121b01..a72e1a7d9f 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json @@ -1,6 +1,5 @@ { "imports": { - "alias:helper": "./helper.ts", - "html-rewriter": "https://ghuc.cc/worker-tools/html-rewriter/index.ts" + "alias:helper": "./helper.ts" } } diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/grandchild-1/index.js b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/grandchild-1/index.js index 59c1e941b1..6a415726e9 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/grandchild-1/index.js +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/grandchild-1/index.js @@ -1,3 +1,3 @@ -import { cwd } from "process" +import { platform } from "process" -export default (input) => `${input}${cwd()}` \ No newline at end of file +export default (input) => `${input}${platform}` \ No newline at end of file diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js new file mode 100644 index 0000000000..35a9b77a4d --- /dev/null +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js @@ -0,0 +1,4 @@ +export default () => { + throw new Error("I should not be loaded") +} + diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json new file mode 100644 index 0000000000..483924df90 --- /dev/null +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json @@ -0,0 +1,5 @@ +{ + "name": "parent-4", + "version": "1.0.0", + "main": "index.js" +} \ No newline at end of file diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/package.json b/packages/edge-bundler/test/fixtures/imports_npm_module/package.json index 3dbc1ca591..3541812a7b 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/package.json +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/package.json @@ -1,3 +1,7 @@ { - "type": "module" + "type": "module", + "dependencies": { + "parent-1": "1.0.0", + "parent-3": "1.0.0" + } } diff --git a/packages/edge-bundler/test/integration/test.js b/packages/edge-bundler/test/integration/test.js index 57e244abd3..227e46bfde 100644 --- a/packages/edge-bundler/test/integration/test.js +++ b/packages/edge-bundler/test/integration/test.js @@ -8,7 +8,7 @@ import { fileURLToPath, pathToFileURL } from 'url' import { promisify } from 'util' import cpy from 'cpy' -import tar from 'tar' +import * as tar from 'tar' import tmp from 'tmp-promise' const exec = promisify(childProcess.exec) diff --git a/packages/edge-bundler/test/util.ts b/packages/edge-bundler/test/util.ts index 2b8d204772..b430c11c68 100644 --- a/packages/edge-bundler/test/util.ts +++ b/packages/edge-bundler/test/util.ts @@ -3,7 +3,9 @@ import { join, resolve } from 'path' import { stderr, stdout } from 'process' import { fileURLToPath, pathToFileURL } from 'url' +import cpy from 'cpy' import { execa } from 'execa' +import * as tar from 'tar' import tmp from 'tmp-promise' import { getLogger } from '../node/logger.js' @@ -17,19 +19,36 @@ const url = new URL(import.meta.url) const dirname = fileURLToPath(url) const fixturesDir = resolve(dirname, '..', 'fixtures') -const useFixture = async (fixtureName: string) => { - const tmpDir = await tmp.dir({ unsafeCleanup: true }) +interface UseFixtureOptions { + copyDirectory?: boolean +} + +const useFixture = async (fixtureName: string, { copyDirectory }: UseFixtureOptions = {}) => { + const tmpDistDir = await tmp.dir({ unsafeCleanup: true }) const fixtureDir = resolve(fixturesDir, fixtureName) - const distPath = join(tmpDir.path, '.netlify', 'edge-functions-dist') + const distPath = join(tmpDistDir.path, '.netlify', 'edge-functions-dist') + + if (copyDirectory) { + const tmpFixtureDir = await tmp.dir({ unsafeCleanup: true }) + + // TODO: Replace with `fs.cp` once we drop support for Node 14. + await cpy(`${fixtureDir}/**`, tmpFixtureDir.path) + + return { + basePath: tmpFixtureDir.path, + cleanup: () => Promise.allSettled([tmpDistDir.cleanup, tmpFixtureDir.cleanup]), + distPath, + } + } return { basePath: fixtureDir, - cleanup: tmpDir.cleanup, + cleanup: tmpDistDir.cleanup, distPath, } } -const inspectFunction = (path: string) => ` +const inspectESZIPFunction = (path: string) => ` import { functions } from "${pathToFileURL(path)}.js"; const responses = {}; @@ -44,6 +63,25 @@ const inspectFunction = (path: string) => ` console.log(JSON.stringify(responses)); ` +const inspectTarballFunction = () => ` +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import manifest from "./___netlify-edge-functions.json" with { type: "json" }; + +const responses = {}; + +for (const functionName in manifest.functions) { + const req = new Request("https://test.netlify"); + const entrypoint = path.resolve(manifest.functions[functionName]); + const func = await import(pathToFileURL(entrypoint)) + const res = await func.default(req); + + responses[functionName] = await res.text(); +} + +console.log(JSON.stringify(responses)); +` + const getRouteMatcher = (manifest: Manifest) => (candidate: string) => manifest.routes.find((route) => { const regex = new RegExp(route.pattern) @@ -69,6 +107,7 @@ const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { const extractCommand = execa('deno', [ 'run', '--allow-all', + '--no-lock', 'https://deno.land/x/eszip@v0.55.2/eszip.ts', 'x', eszipPath, @@ -99,7 +138,37 @@ const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { await fs.rename(stage2Path, `${stage2Path}.js`) // Run function that imports the extracted stage 2 and invokes each function. - const evalCommand = execa('deno', ['eval', '--no-check', '--import-map', importMapPath, inspectFunction(stage2Path)]) + const evalCommand = execa('deno', [ + 'eval', + '--no-check', + '--import-map', + importMapPath, + inspectESZIPFunction(stage2Path), + ]) + + evalCommand.stderr?.pipe(stderr) + + const result = await evalCommand + + await tmpDir.cleanup() + + return JSON.parse(result.stdout) +} + +const runTarball = async (tarballPath: string) => { + const tmpDir = await tmp.dir({ unsafeCleanup: true }) + + await tar.extract({ + cwd: tmpDir.path, + file: tarballPath, + }) + + const evalCommand = execa('deno', ['eval', inspectTarballFunction()], { + cwd: join(tmpDir.path, 'src'), + env: { + DENO_DIR: '../deno_dir', + }, + }) evalCommand.stderr?.pipe(stderr) @@ -110,4 +179,4 @@ const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { return JSON.parse(result.stdout) } -export { fixturesDir, getRouteMatcher, testLogger, runESZIP, useFixture } +export { fixturesDir, getRouteMatcher, testLogger, runESZIP, runTarball, useFixture }