Merge pull request #845 from OpenFn/nodes-and-edges-release

[epic] Nodes and edges release
OpenFn · Jul 24, 2023 · 07a5ae0 · 07a5ae0
2 parents 9e03f48 + f072554
commit 07a5ae0
Show file tree

Hide file tree

Showing 187 changed files with 9,422 additions and 5,189 deletions.
diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs
@@ -0,0 +1,5 @@
+[
+  # Remove when next version of postgrex is released.
+  # https://github.com/elixir-ecto/postgrex/issues/651
+  {"deps/postgrex/lib/postgrex/type_module.ex", :improper_list_constr}
+]
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ lightning-*.tar
 # In case you use Node.js/npm, you want to ignore these.
 npm-debug.log
 /assets/node_modules/
+/examples/**/node_modules/
 
 # coveralls output file
 /excoveralls.json

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,11 @@ and this project adheres to
 
 ### Added
 
+- Write/run sql script to convert triggers
+  [#875](https://github.com/OpenFn/Lightning/issues/875)
+- Export projects as `.yaml` via UI
+  [#249](https://github.com/OpenFn/Lightning/issues/249)
+
 ### Changed
 
 ### Fixed

diff --git a/assets/.postcssrc b/assets/.postcssrc
@@ -0,0 +1,5 @@
+{
+  "plugins": {
+    "tailwindcss": {}
+  }
+}
diff --git a/assets/css/app.css b/assets/css/app.css
@@ -2,6 +2,8 @@
 @import 'tailwindcss/components';
 @import 'tailwindcss/utilities';
 @import './monaco-style-overrides.css';
+@import 'reactflow/dist/style.css';
+@import '../../deps/petal_components/assets/default.css';
 /* This file is for your main application CSS */
 
 /* Alerts and form errors used by phx.new */

diff --git a/assets/dev-server.mjs b/assets/dev-server.mjs
@@ -0,0 +1,31 @@
+import path from 'node:path';
+import esbuild from 'esbuild';
+import { copy } from 'esbuild-plugin-copy';
+import postcss from 'esbuild-postcss';
+
+const ctx = await esbuild.context({
+  // absWorkingDir: path.resolve('dev-server'),
+  entryPoints: ['dev-server/src/index.tsx'],
+  outdir: 'dev-server/dist/',
+  bundle: true,
+  splitting: true,
+  sourcemap: true,
+  format: 'esm',
+  target: ['es2020'],
+  plugins: [
+    postcss(),
+    copy({
+      assets: {
+        from: ['./dev-server/public/*'],
+        to: ['.'],
+      },
+    }),
+  ],
+});
+
+await ctx.watch();
+
+await ctx.serve({
+  servedir: 'dev-server/dist',
+  port: 3000,
+});
diff --git a/assets/dev-server/.gitignore b/assets/dev-server/.gitignore
@@ -0,0 +1 @@
+dist
diff --git a/assets/dev-server/public/index.html b/assets/dev-server/public/index.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Workflow Test Harness</title>
+    <link rel="stylesheet" href="index.css"> 
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="importmap">
+      {
+        "imports": {
+          "react": "./react.js",
+          "react/jsx-runtime": "./react.js",
+          "react-dom": "./react.js",
+          "react-dom/client": "./react.js"
+        }
+      }
+    </script>
+    <script src="react.js" type="module"></script>
+    <script src="index.js" type="module"></script>
+  </body>
+</html>
diff --git a/assets/dev-server/src/App.tsx b/assets/dev-server/src/App.tsx
@@ -0,0 +1,154 @@
+import React, { useState, useEffect, useCallback, useRef } from 'react';
+
+import WorkflowDiagram from '../../js/workflow-diagram/WorkflowDiagram'
+// import useStore from './store'
+import { createWorkflowStore } from '../../js/workflow-editor/store'
+import workflows from './workflows';
+import './main.css'
+import 'reactflow/dist/style.css';
+
+const Form = ({ nodeId, store, onChange }) => {
+  if (!nodeId) {
+    return <div>Nothing selected</div>
+  }
+  const { jobs, edges, triggers } = store.getState();
+  const item = jobs.find(({ id }) => id == nodeId) || edges.find(({ id }) => id == nodeId) || triggers.find(({ id }) => id == nodeId) ;
+  return  (<>
+    <p>
+      <span>Name</span>
+      <input
+        value={item.name || item.id}
+        className="border-1 border-slate-200 ml-2 p-2"
+        onChange={(evt) => onChange({ name: evt.target.value })} />
+    </p>
+    {item.adaptor && <p>{`adaptor: ${item.adaptor}`}</p>}
+    {item.type && <p>{`type: ${item.type}`}</p>}
+    {item.target_job_id ?
+     <p>{`condition: ${item.condition}`}</p>
+     : <p>{`expression: ${item.cronExpression || item.expression}`}</p>}
+  </>);
+}
+
+export default () => {
+  const [workflowId, setWorkflowId] = useState('chart1');
+  const [history, setHistory ] = useState([])
+  const [store, setStore ] = useState({});
+  const [selectedId, setSelectedId] = useState<string>();
+  const ref = useRef(null)
+
+  const [workflow, setWorkflow] = useState({ jobs: [], triggers: [], edges: [] })
+
+  // on startup (or on workflow id change) create a store
+  // on change, set the state back into the app.
+  // Now if the store changes, we can deal with it
+  useEffect(() => {
+    const onChange = (evt) => {
+      // what do we do on change, and how do we call this safely?
+      console.log('CHANGE', evt.id, evt.patches)
+      setHistory((h) => [evt, ...h])
+    }
+
+    const s = createWorkflowStore(workflows[workflowId], onChange)
+
+    const unsubscribe = s.subscribe(({ jobs, edges, triggers }) => {
+      console.log('store change: ', { jobs, edges, triggers })
+      setWorkflow({ jobs, edges, triggers });
+    });
+
+    const { jobs, edges, triggers } = s.getState();
+    // Set the chart to null to reset its positions
+    setWorkflow({ jobs: [], edges: [], triggers: [] });
+
+    // now set the chart properly
+    // use a timeout to make sure its applied
+    setTimeout( () =>  {
+      setWorkflow({ jobs, edges, triggers });
+      setStore(s);
+    }, 1)
+
+    return () => unsubscribe();
+  }, [workflowId])
+
+  const handleSelectionChange = useCallback((id: string) => {
+    setSelectedId(id)
+  }, [])
+
+  const handleNameChange = useCallback(({ name }) => {
+    const { jobs, edges, change } = store.getState();
+    const diff = { name, placeholder: false };
+
+    const node = jobs.find(j => j.id === selectedId);
+    if (node.placeholder) {
+      diff.placeholder = false;
+
+      const edge = edges.find(e => e.target_job_id === selectedId);
+      change(edge.id, 'edges', { placeholder: false } );
+    }
+
+    change(selectedId, 'jobs', diff);
+
+  }, [store, selectedId])
+
+  // Adding a job in the store will push the new workflow structure through to the inner component
+  // Selection should be preserved (but probably isn't right now)
+  // At the moment this doesn't animate, which is fine and expected
+  const addJob = useCallback(() => {
+    const { add } = store.getState();
+
+    const newNodeId = crypto.randomUUID();
+    add({
+      jobs: [{
+        id: newNodeId,
+        type: 'job',
+      }],
+      edges: [{
+        source_job_id: selectedId?.id ?? 'a', target_job_id: newNodeId
+      }]
+    })
+  }, [store, selectedId]);
+
+  const handleRequestChange = useCallback((diff) => {
+    const { add } = store.getState();
+    add(diff)
+  }, [store]);
+
+  return (<div className="flex flex-row h-full w-full">
+    <div className="flex-1 border-2 border-slate-200 m-2 bg-secondary-100" ref={ref}>
+      <WorkflowDiagram 
+        ref={ref.current}
+        workflow={workflow}
+        requestChange={handleRequestChange}
+        onSelectionChange={handleSelectionChange}
+      />
+    </div>
+    <div className="flex-1 flex flex-col h-full w-1/3">
+      <div className="border-2 border-slate-200 m-2 p-2">
+        {/*
+          Options to control data flow from outside the chart
+          These must write to the store and push to the component
+        */}
+        <button className="bg-primary-500 mx-2 py-2 px-4 border border-transparent shadow-sm rounded-md text-white" onClick={() => setWorkflowId('chart1')}>Workflow 1</button>
+        <button className="bg-primary-500 mx-2 py-2 px-4 border border-transparent shadow-sm rounded-md text-white" onClick={() => setWorkflowId('chart2')}>Workflow 2</button>
+        <button className="bg-primary-500 mx-2 py-2 px-4 border border-transparent shadow-sm rounded-md text-white" onClick={() => addJob()}>Add Job</button>
+      </div>
+      <div className="flex-1 border-2 border-slate-200 m-2 p-2">
+        <h2 className="text-center">Selected</h2>
+        <Form store={store} nodeId={selectedId} onChange={handleNameChange}/>
+      </div>
+      <div className="flex-1 border-2 border-slate-200 m-2 p-2 overflow-y-auto">
+        <h2 className="text-center">Change Events</h2>
+        <ul className="ml-4">{
+          history.map((change) => {
+          return (<li key={change.id} className="border border-slate-50 border-1 p-4 m-2">
+            <h3>{change.id}</h3>
+            <ul className="list-disc ml-4">
+              {change.patches.map((p) => <li key={p.path}>{`${p.op} ${p.path}`}</li>)}
+            </ul>
+          </li>)
+          })
+        }</ul>
+      </div>
+    </div>
+  </div>
+  );
+};
diff --git a/assets/dev-server/src/index.tsx b/assets/dev-server/src/index.tsx
@@ -0,0 +1,13 @@
+import React, { StrictMode } from "react";
+import { createRoot } from "react-dom/client";
+
+import App from "./App";
+
+new EventSource('/esbuild').addEventListener('change', () => location.reload())
+
+const root = createRoot(document.getElementById("root"));
+root.render(
+  <StrictMode>
+    <App />
+  </StrictMode>
+);
diff --git a/assets/dev-server/src/main.css b/assets/dev-server/src/main.css
@@ -0,0 +1,7 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+#root {
+  height: 100vh;
+}
diff --git a/assets/dev-server/src/workflows.ts b/assets/dev-server/src/workflows.ts
@@ -0,0 +1,72 @@
+const workflows = {
+  chart1: {
+    id: 'chart1',
+    jobs: [
+      {
+        id: 'a',
+        name: 'Do the thing',
+        adaptor: 'common',
+        expression: 'fn(s => s)',
+      },
+      {
+        id: 'b',
+        adaptor: 'salesforce',
+        expression: 'fn(s => s)',
+      },
+      {
+        id: 'c',
+        adaptor: 'http',
+        expression: 'fn(s => s)',
+      },
+    ],
+    triggers: [
+      {
+        id: 'z',
+        type: 'cron',
+        cronExpression: '0 0 0',
+      },
+    ],
+    edges: [
+      {
+        id: 'z-a',
+        name: 'on success',
+        source_trigger_id: 'z',
+        target_job_id: 'a',
+      },
+      {
+        id: 'a-b',
+        name: 'on success',
+        source_job_id: 'a',
+        target_job_id: 'b',
+      },
+      {
+        id: 'a-c',
+        name: 'on success',
+        source_job_id: 'a',
+        target_job_id: 'c',
+      },
+    ],
+  },
+  chart2: {
+    id: 'chart2',
+    jobs: [{ id: 'a' }],
+    triggers: [{ id: 'z' }],
+    edges: [{ id: 'z-a', source_trigger_id: 'z', target_job_id: 'a' }],
+  },
+  chart3: {
+    id: 'chart3',
+    jobs: [
+      { id: 'a' },
+      { id: 'b', label: 'this is a very long node name oh yes' },
+      { id: 'c' },
+    ],
+    triggers: [],
+    edges: [
+      // { id: 'z-a', source_trigger_id: 'z', target_job_id: 'a' },
+      { id: 'a-b', source_job: 'a', target_job_id: 'b' },
+      { id: 'b-c', source_job: 'b', target_job_id: 'c' },
+    ],
+  },
+};
+
+export default workflows;