Merge pull request #61 from OpenFn/tidy-project

Tidy project
OpenFn · Nov 8, 2022 · eb52b0d · eb52b0d
2 parents 75fa8ec + 6c57303
commit eb52b0d
Show file tree

Hide file tree

Showing 24 changed files with 272 additions and 299 deletions.
diff --git a/.changeset/sour-points-tie.md b/.changeset/sour-points-tie.md
@@ -0,0 +1,5 @@
+---
+'@openfn/describe-package': patch
+---
+
+Include worker bundle in package
diff --git a/README.md b/README.md
@@ -1,13 +1,46 @@
-Lightning ProjectSpace
+OpenFn Kit
 ======================
-
 [![CircleCI](https://dl.circleci.com/status-badge/img/gh/OpenFn/kit/tree/main.svg?style=shield)](https://dl.circleci.com/status-badge/redirect/gh/OpenFn/kit/tree/main)
 
+**Kit** _noun_ _/kɪt/_
+
+A set of articles or equipment needed for a specific purpose.
+
+1. _a football kit_
+1. _the next-generation openfn data integration kit_
+
+---
+
+This repo contains runtime, tooling, libraries and components to support the next generation core openfn data integration pipeline.
+
+It is a kitbag of Javascript-based components to support Lightning.
+
+## Prerequisities
+
+* [asdf](https://github.com/asdf-vm/asdf)
+* [pnpm](https://pnpm.io/installation)
+
+We use [asdf](https://github.com/asdf-vm/asdf) to configure our local
+environments and ensure consistency of versions.
+
+You should install asdf and the [NodeJs](https://github.com/asdf-vm/asdf-nodejs) plugin.
+
+We use [`pnpm`](https://pnpm.io/installation), a fast, disk space efficient package manager, to handle node dependencies within the repo.
+
 ## Installing
 
-- Install [`pnpm`](https://pnpm.io/installation)
-- Run `pnpm run setup`
-- Run `pnpm run build`
+- `$ pnpm run setup`
+- `$ pnpm build`
+
+## Running Tests
+
+```
+pnpm run test
+```
+
+# Development Guide
+
+Thanks for being here! You're contributing to a digital public good that will always be free and open source and aimed at serving innovative NGOs, governments, and social impact organizations the world over! You rock ❤️
 
 ## Releases & Changesets
 
@@ -91,35 +124,6 @@ Run the install command as printed in your shell - something like `npm -g dist/o
 
 You can run `openfn test` to exercise the runtime and compiler.
 
-
-## Packages
-
-- [`@openfn/describe-package`](packages/describe-package)  
-- [`@openfn/workflow-diagram`](packages/workflow-diagram)
-
-## Examples
-
-The example apps serve to illustrate how these packages can be used, and also
-for development, any changes detected in the dependencies will trigger a rebuild in the example.
-
-**ProjectSpace Flow**
-
-```
-pnpm run -C examples/flow start
-```
-
-**Compiler Worker**
-
-```
-pnpm run -C examples/compiler-worker start
-```
-
-## Running Tests
-
-```
-pnpm run test
-```
-
 ## Documentation
 
 For information on the history of the OpenFn internals and ideas for the future

diff --git a/examples/README.md b/examples/README.md
@@ -0,0 +1,22 @@
+## Examples
+
+The example apps serve to illustrate how these packages can be used, and also
+for development, any changes detected in the dependencies will trigger a rebuild in the example.
+
+**ProjectSpace Flow**
+
+Uses packages/workflow-diagram
+
+From root:
+```
+pnpm run -C examples/flow start
+```
+
+**DTS Inspector**
+
+Uses packages/describe-package
+
+From root:
+```
+pnpm run -C examples/describe-package start
+```
diff --git a/examples/compiler-worker/.postcssrc → examples/dts-inspector/.postcssrc b/examples/compiler-worker/.postcssrc → examples/dts-inspector/.postcssrc
diff --git a/examples/compiler-worker/CHANGELOG.md → examples/dts-inspector/CHANGELOG.md b/examples/compiler-worker/CHANGELOG.md → examples/dts-inspector/CHANGELOG.md
@@ -1,4 +1,4 @@
-# compiler-worker
+# dts-inspector
 
 ## 1.0.5
 

diff --git a/examples/compiler-worker/README.md → examples/dts-inspector/README.md b/examples/compiler-worker/README.md → examples/dts-inspector/README.md
@@ -1,7 +1,7 @@
-# Describe Package Worker
+# DTS Inspector
 
 A living example of how to use the `@openfn/describe-package` worker in
-a browser.
+a browser to inspect an openfn language adaptor.
 
 ## Running
 

diff --git a/examples/compiler-worker/esbuild.js → examples/dts-inspector/esbuild.js b/examples/compiler-worker/esbuild.js → examples/dts-inspector/esbuild.js
diff --git a/examples/compiler-worker/index.html → examples/dts-inspector/index.html b/examples/compiler-worker/index.html → examples/dts-inspector/index.html
@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8"/>
-    <title>Workflow Diagram Demo</title>
+    <title>DTS Inspector Demo</title>
    	<link rel="stylesheet" href="/dist/src/index.css"> 
   </head>
   <body class="bg-slate-200">

diff --git a/examples/compiler-worker/package.json → examples/dts-inspector/package.json b/examples/compiler-worker/package.json → examples/dts-inspector/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "compiler-worker",
+  "name": "dts-inspector",
   "version": "1.0.5",
   "description": "",
   "main": "index.js",

diff --git a/examples/compiler-worker/src/app.css → examples/dts-inspector/src/app.css b/examples/compiler-worker/src/app.css → examples/dts-inspector/src/app.css
diff --git a/examples/compiler-worker/src/dts-area.tsx → examples/dts-inspector/src/dts-area.tsx b/examples/compiler-worker/src/dts-area.tsx → examples/dts-inspector/src/dts-area.tsx
diff --git a/examples/compiler-worker/src/from-unpkg.tsx → examples/dts-inspector/src/from-unpkg.tsx b/examples/compiler-worker/src/from-unpkg.tsx → examples/dts-inspector/src/from-unpkg.tsx
@@ -3,7 +3,7 @@ import { createRoot } from "react-dom/client";
 
 import "./app.css";
 import { StatusIcon, FuncIcon } from "./icons";
-import { Pack, Project, describeDts } from "@openfn/compiler";
+import { Pack, Project, describeDts } from "@openfn/describe-package";
 
 const packageOrDts = /(?:package.json)|(?:\.d\.ts$)/i;
 const moduleOptions = ["@openfn/[email protected]"];

diff --git a/examples/compiler-worker/src/icons.tsx → examples/dts-inspector/src/icons.tsx b/examples/compiler-worker/src/icons.tsx → examples/dts-inspector/src/icons.tsx
diff --git a/examples/compiler-worker/src/index.tsx → examples/dts-inspector/src/index.tsx b/examples/compiler-worker/src/index.tsx → examples/dts-inspector/src/index.tsx
diff --git a/examples/compiler-worker/tailwind.config.cjs → examples/dts-inspector/tailwind.config.cjs b/examples/compiler-worker/tailwind.config.cjs → examples/dts-inspector/tailwind.config.cjs
diff --git a/packages/describe-package/esbuild-worker.ts b/packages/describe-package/esbuild-worker.ts
@@ -0,0 +1,93 @@
+import { build } from 'esbuild';
+import path from 'path';
+import { readFile, rm } from 'fs/promises';
+import { BuildOptions } from 'esbuild';
+
+export default function rawPlugin() {
+  return {
+    name: 'raw',
+    setup(build) {
+      build.onResolve({ filter: /\?raw$/ }, (args) => {
+        return {
+          path: path.isAbsolute(args.path)
+            ? args.path
+            : path.join(args.resolveDir, args.path),
+          namespace: 'raw-loader',
+        };
+      });
+      build.onLoad(
+        { filter: /\?raw$/, namespace: 'raw-loader' },
+        async (args) => {
+          return {
+            contents: await readFile(args.path.replace(/\?raw$/, '')),
+            loader: 'text',
+          };
+        }
+      );
+    },
+  };
+}
+
+// esbuild watch in dev mode to rebuild out files
+const watchOptions = {
+  onRebuild(error) {
+    if (error)
+      console.error('esbuild: Watch build failed:', error.getMessage());
+    else console.log('esbuild: Watch build succeeded');
+  },
+};
+
+let watch = process.argv[2] === 'watch' ? watchOptions : false;
+
+const commonBuildOptions: BuildOptions = {
+  bundle: true,
+  write: true,
+  watch,
+  format: 'esm',
+  target: ['es2020'],
+  outdir: './dist',
+  external: ['fs', 'events', 'stream', 'path', 'util', 'constants', 'assert'],
+  pure: ['console.log', 'console.time', 'console.timeEnd'],
+  sourcemap: false,
+};
+
+try {
+  /**
+   * WebWorker internals modules
+   * This is the bundle that includes the Worker, Typescript and the interface
+   * to query and interact with the Compiler. In order to provide a single file
+   * for using the library we build just the worker, and later inject it into
+   * the Worker entrypoint.
+   */
+  await build({
+    ...commonBuildOptions,
+    entryPoints: {
+      'worker-internals': './src/worker/worker.ts',
+    },
+    format: 'esm',
+    minify: true,
+  });
+
+  /**
+   * WebWorker Entrypoint
+   * This is the one that actually gets used in the browser, note the `rawPlugin`
+   * which will load in the output of the `worker-internals` file as a string
+   * into the entrypoint - allowing us to bundle both the worker code and the
+   * entrypoint in the same file.
+   */
+  await build({
+    ...commonBuildOptions,
+    entryPoints: {
+      worker: './src/worker/index.ts',
+    },
+    format: 'esm',
+    minify: false,
+    plugins: [rawPlugin()],
+  });
+
+  // Cleanup worker-internals since they are bundled into the worker.
+  await rm('./dist/worker-internals.js');
+} catch (error) {
+  console.error(error);
+  process.exit(1);
+}
diff --git a/packages/describe-package/esbuild.ts b/packages/describe-package/esbuild.ts