Use workers to process files in parallel (#280)

* Bump @types/node to 20.x.x and increase 'engines' check * Add dependency on piscina * Add a worker pool --------- Co-authored-by: Walker Burgin <[email protected]>
swc-project · Jan 24, 2024 · 7d9a493 · 7d9a493
1 parent 908d0a1
commit 7d9a493
Show file tree

Hide file tree

Showing 9 changed files with 159 additions and 64 deletions.
diff --git a/package.json b/package.json
@@ -38,7 +38,7 @@
   },
   "homepage": "https://github.com/swc-project/cli#readme",
   "engines": {
-    "node": ">= 12.13"
+    "node": ">= 16.14.0"
   },
   "bin": {
     "swc": "./bin/swc.js",
@@ -50,6 +50,7 @@
     "commander": "^7.1.0",
     "fast-glob": "^3.2.5",
     "minimatch": "^9.0.3",
+    "piscina": "^4.3.0",
     "semver": "^7.3.8",
     "slash": "3.0.0",
     "source-map": "^0.7.3"
@@ -59,7 +60,7 @@
     "@swc/core": "^1.2.66",
     "@swc/jest": "^0.1.2",
     "@types/jest": "^29.5.0",
-    "@types/node": "^12.19.16",
+    "@types/node": "^20.11.5",
     "@types/semver": "^7.3.13",
     "chokidar": "^3.5.1",
     "deepmerge": "^4.2.2",

diff --git a/src/spack/index.ts b/src/spack/index.ts
@@ -56,7 +56,7 @@ const makeDir = promisify(mkdir);
         await makeDir(dirname(fullPath), { recursive: true });
         await write(fullPath, output[name].code, "utf-8");
         if (output[name].map) {
-          await write(`${fullPath}.map`, output[name].map, "utf-8");
+          await write(`${fullPath}.map`, output[name].map!, "utf-8");
         }
       });
     } else {

diff --git a/src/swc/__tests__/options.test.ts b/src/swc/__tests__/options.test.ts
@@ -130,6 +130,32 @@ describe("parserArgs", () => {
       expect(mockExit).toHaveBeenCalledWith(2);
       expect(mockConsoleError).toHaveBeenCalledTimes(2);
     });
+
+    it("--workers exits on non-numeric values", async () => {
+      const args = [
+        "node",
+        "/path/to/node_modules/swc-cli/bin/swc.js",
+        "--workers",
+        "not-a-number",
+        "src",
+      ];
+      await parserArgs(args);
+      expect(mockExit).toHaveBeenCalledWith(2);
+      expect(mockConsoleError).toHaveBeenCalledTimes(2);
+    });
+
+    it("--workers exits on non-integer values", async () => {
+      const args = [
+        "node",
+        "/path/to/node_modules/swc-cli/bin/swc.js",
+        "--workers",
+        "1.5",
+        "src",
+      ];
+      await parserArgs(args);
+      expect(mockExit).toHaveBeenCalledWith(2);
+      expect(mockConsoleError).toHaveBeenCalledTimes(2);
+    });
   });
 
   describe("--source-maps", () => {

diff --git a/src/swc/compile.ts b/src/swc/compile.ts
@@ -62,7 +62,7 @@ export async function outputResult(
   } else {
     await Promise.all([
       writeFile(destFile, sourceCode, { mode }),
-      writeFile(sourceMapPath, sourceMap, { mode }),
+      writeFile(sourceMapPath, sourceMap!, { mode }),
     ]);
   }
 }

diff --git a/src/swc/dir.ts b/src/swc/dir.ts
@@ -1,10 +1,10 @@
-import slash from "slash";
 import { existsSync, promises } from "fs";
-import { dirname, relative, join } from "path";
+import { dirname, resolve } from "path";
+import Piscina from "piscina";
 import { CompileStatus } from "./constants";
 import { CliOptions } from "./options";
-import { compile, exists } from "./util";
-import { outputResult } from "./compile";
+import { exists, getDest } from "./util";
+import handleCompile from "./dirWorker";
 import {
   globSources,
   isCompilableExtension,
@@ -26,53 +26,8 @@ declare module "fs" {
 
 const { mkdir, rmdir, rm, copyFile, unlink } = promises;
 
-const cwd = process.cwd();
 const recursive = { recursive: true };
 
-/**
- * Removes the leading directory, including all parent relative paths
- */
-function stripComponents(filename: string) {
-  const components = filename.split("/").slice(1);
-  if (!components.length) {
-    return filename;
-  }
-  while (components[0] === "..") {
-    components.shift();
-  }
-  return components.join("/");
-}
-
-function getDest(filename: string, outDir: string, ext?: string) {
-  const relativePath = slash(relative(cwd, filename));
-  let base = stripComponents(relativePath);
-  if (ext) {
-    base = base.replace(/\.\w*$/, ext);
-  }
-  return join(outDir, base);
-}
-
-async function handleCompile(
-  filename: string,
-  outDir: string,
-  sync: boolean,
-  swcOptions: Options
-) {
-  const dest = getDest(filename, outDir, ".js");
-  const sourceFileName = slash(relative(dirname(dest), filename));
-
-  const options = { ...swcOptions, sourceFileName };
-
-  const result = await compile(filename, options, sync, dest);
-
-  if (result) {
-    await outputResult(result, filename, dest, options);
-    return CompileStatus.Compiled;
-  } else {
-    return CompileStatus.Omitted;
-  }
-}
-
 async function handleCopy(filename: string, outDir: string) {
   const dest = getDest(filename, outDir);
   const dir = dirname(dest);
@@ -126,7 +81,12 @@ async function initialCompilation(cliOptions: CliOptions, swcOptions: Options) {
   if (sync) {
     for (const filename of compilable) {
       try {
-        const result = await handleCompile(filename, outDir, sync, swcOptions);
+        const result = await handleCompile({
+          filename,
+          outDir,
+          sync,
+          swcOptions,
+        });
         results.set(filename, result);
       } catch (err: any) {
         console.error(err.message);
@@ -143,10 +103,16 @@ async function initialCompilation(cliOptions: CliOptions, swcOptions: Options) {
       }
     }
   } else {
+    const workers = new Piscina({
+      filename: resolve(__dirname, "./dirWorker.js"),
+      maxThreads: cliOptions.workers,
+      concurrentTasksPerWorker: 2,
+    });
+
     await Promise.all([
       Promise.allSettled(
-        compilable.map(file =>
-          handleCompile(file, outDir, sync, swcOptions).catch(err => {
+        compilable.map(filename =>
+          workers.run({ filename, outDir, sync, swcOptions }).catch(err => {
             console.error(err.message);
             throw err;
           })
@@ -264,12 +230,12 @@ async function watchCompilation(cliOptions: CliOptions, swcOptions: Options) {
       if (isCompilableExtension(filename, extensions)) {
         try {
           const start = process.hrtime();
-          const result = await handleCompile(
+          const result = await handleCompile({
             filename,
             outDir,
             sync,
-            swcOptions
-          );
+            swcOptions,
+          });
           if (!quiet && result === CompileStatus.Compiled) {
             const end = process.hrtime(start);
             console.log(

diff --git a/src/swc/dirWorker.ts b/src/swc/dirWorker.ts
@@ -0,0 +1,28 @@
+import slash from "slash";
+import { dirname, relative } from "path";
+import { CompileStatus } from "./constants";
+import { compile, getDest } from "./util";
+import { outputResult } from "./compile";
+
+import type { Options } from "@swc/core";
+
+export default async function handleCompile(opts: {
+  filename: string;
+  outDir: string;
+  sync: boolean;
+  swcOptions: Options;
+}) {
+  const dest = getDest(opts.filename, opts.outDir, ".js");
+  const sourceFileName = slash(relative(dirname(dest), opts.filename));
+
+  const options = { ...opts.swcOptions, sourceFileName };
+
+  const result = await compile(opts.filename, options, opts.sync, dest);
+
+  if (result) {
+    await outputResult(result, opts.filename, dest, options);
+    return CompileStatus.Compiled;
+  } else {
+    return CompileStatus.Omitted;
+  }
+}
diff --git a/src/swc/options.ts b/src/swc/options.ts
@@ -102,6 +102,11 @@ export const initProgram = () => {
     collect
   );
 
+  program.option(
+    "--workers [number]",
+    "The number of workers to use for parallel processing"
+  );
+
   program.option(
     "--log-watch-compilation",
     "Log a message when a watched file is successfully compiled",
@@ -157,6 +162,7 @@ export interface CliOptions {
    * Invoke swc using transformSync. It's useful for debugging.
    */
   readonly sync: boolean;
+  readonly workers: number | undefined;
   readonly sourceMapTarget?: string;
   readonly filename: string;
   readonly filenames: string[];
@@ -207,6 +213,16 @@ export default function parserArgs(args: string[]) {
     );
   }
 
+  let workers: number | undefined;
+  if (opts.workers != null) {
+    workers = parseFloat(opts.workers);
+    if (!Number.isInteger(workers) || workers < 0) {
+      errors.push(
+        "--workers must be a positive integer (found " + opts.workers + ")"
+      );
+    }
+  }
+
   if (errors.length) {
     console.error("swc:");
     for (const error of errors) {
@@ -265,6 +281,7 @@ export default function parserArgs(args: string[]) {
     filename: opts.filename,
     filenames,
     sync: !!opts.sync,
+    workers,
     sourceMapTarget: opts.sourceMapTarget,
     extensions: opts.extensions || DEFAULT_EXTENSIONS,
     watch: !!opts.watch,

diff --git a/src/swc/util.ts b/src/swc/util.ts
@@ -1,7 +1,7 @@
 import * as swc from "@swc/core";
 import slash from "slash";
 import { mkdirSync, writeFileSync, promises } from "fs";
-import { dirname, relative } from "path";
+import { dirname, join, relative } from "path";
 
 export async function exists(path: string): Promise<boolean> {
   let pathExists = true;
@@ -125,3 +125,28 @@ export function assertCompilationResult<T>(
     );
   }
 }
+
+/**
+ * Removes the leading directory, including all parent relative paths
+ */
+function stripComponents(filename: string) {
+  const components = filename.split("/").slice(1);
+  if (!components.length) {
+    return filename;
+  }
+  while (components[0] === "..") {
+    components.shift();
+  }
+  return components.join("/");
+}
+
+const cwd = process.cwd();
+
+export function getDest(filename: string, outDir: string, ext?: string) {
+  const relativePath = slash(relative(cwd, filename));
+  let base = stripComponents(relativePath);
+  if (ext) {
+    base = base.replace(/\.\w*$/, ext);
+  }
+  return join(outDir, base);
+}
diff --git a/yarn.lock b/yarn.lock
@@ -795,10 +795,12 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.3.tgz#f0b991c32cfc6a4e7f3399d6cb4b8cf9a0315014"
   integrity sha512-p6ua9zBxz5otCmbpb5D3U4B5Nanw6Pk3PPyX05xnxbB/fRv71N7CPmORg7uAD5P70T0xmx1pzAx/FUfa5X+3cw==
 
-"@types/node@^12.19.16":
-  version "12.20.55"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240"
-  integrity sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==
+"@types/node@^20.11.5":
+  version "20.11.5"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.5.tgz#be10c622ca7fcaa3cf226cf80166abc31389d86e"
+  integrity sha512-g557vgQjUUfN76MZAN/dt1z3dzcUsimuysco0KeluHgrPdJXkP/XdAURgyO2W9fZWHRtRBiVKzKn8vyOAwlG+w==
+  dependencies:
+    undici-types "~5.26.4"
 
 "@types/prettier@^2.1.5":
   version "2.7.2"
@@ -2260,6 +2262,24 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
 
+nice-napi@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/nice-napi/-/nice-napi-1.0.2.tgz#dc0ab5a1eac20ce548802fc5686eaa6bc654927b"
+  integrity sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==
+  dependencies:
+    node-addon-api "^3.0.0"
+    node-gyp-build "^4.2.2"
+
+node-addon-api@^3.0.0:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
+  integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
+
+node-gyp-build@^4.2.2:
+  version "4.8.0"
+  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.0.tgz#3fee9c1731df4581a3f9ead74664369ff00d26dd"
+  integrity sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
@@ -2411,6 +2431,13 @@ pirates@^4.0.4:
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b"
   integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==
 
+piscina@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/piscina/-/piscina-4.3.0.tgz#fd219f507d410c61dbfb9bd4155c1f19eddb8535"
+  integrity sha512-vTQszGZj78p0BHFNO/cSvpzPUYa4tLXRe30aIYyQjqRS3fK/kPqdxvkTfGXQlEpWOI+mOOkda0iEY6NaanLWJA==
+  optionalDependencies:
+    nice-napi "^1.0.2"
+
 pkg-dir@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3"
@@ -2828,6 +2855,11 @@ typescript@~4.3.2:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4"
   integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==
 
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
+
 update-browserslist-db@^1.0.10:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3"