macro nodes (#78)

core/src/flow-schema.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { VisualNode, NodeDefinition, Node } from "./node";
+import { VisualNode, NodeDefinition, Node, ResolvedVisualNode } from "./node";
 
 const importSchema = z.record(z.string(), z.string().or(z.array(z.string())));
 const position = z.strictObject({ x: z.number(), y: z.number() });
@@ -33,11 +33,20 @@ const instance = z
     visibleOutputs: z.optional(z.array(z.string())),
     nodeId: z.optional(z.string()),
     node: z.optional(z.any()),
+    macroId: z.optional(z.string()),
+    macroData: z.optional(z.any()),
     style: z.optional(nodeStyle),
   })
-  .refine((val) => val.node || val.nodeId, {
-    message: "Instance must have either an inline node or refer to a nodeId",
-  });
+  .refine(
+    (val) =>
+      val.node ||
+      val.nodeId ||
+      (val.macroId && typeof val.macroData !== "undefined"),
+    {
+      message:
+        "Instance must have either an inline node or refer to a nodeId, or be a macro instance",
+    }
+  );
 
 const inputPinSchema = z.union([
   z.string(),
@@ -116,14 +125,14 @@ export type ResolvedDependenciesDefinitions = Record<
 >;
 
 export type ResolvedFlydeFlowDefinition = {
-  main: VisualNode;
+  main: ResolvedVisualNode;
   dependencies: ResolvedDependenciesDefinitions;
 };
 
 export type ResolvedDependencies = Record<string, ImportedNode>;
 
 export type ResolvedFlydeRuntimeFlow = {
-  main: VisualNode;
+  main: ResolvedVisualNode;
   dependencies: ResolvedDependencies;
 };
 

core/src/node/node-instance.ts

@@ -1,4 +1,13 @@
-import { InputPinsConfig, Node, NodeDefinition, NodeStyle, Pos } from "..";
+import {
+  CodeNode,
+  InputPinsConfig,
+  Node,
+  NodeDefinition,
+  NodeStyle,
+  Pos,
+  ResolvedVisualNode,
+  VisualNode,
+} from "..";
 import { slug } from "cuid";
 
 export interface NodeInstanceConfig {
@@ -7,20 +16,42 @@ export interface NodeInstanceConfig {
   visibleOutputs?: string[];
   displayName?: string;
   style?: NodeStyle;
+  id: string;
+  pos: Pos;
 }
 
 export interface RefNodeInstance extends NodeInstanceConfig {
-  id: string;
   nodeId: string;
-  pos: Pos;
 }
 
 export interface InlineNodeInstance extends NodeInstanceConfig {
-  id: string;
-  node: Node;
-  pos: Pos;
+  node: VisualNode | CodeNode;
+}
+
+export interface ResolvedInlineNodeInstance extends NodeInstanceConfig {
+  node: ResolvedVisualNode | CodeNode;
+}
+
+export interface MacroNodeInstance extends NodeInstanceConfig {
+  macroId: string;
+  macroData: any;
 }
-export type NodeInstance = RefNodeInstance | InlineNodeInstance;
+
+export interface ResolvedMacroNodeInstance extends NodeInstanceConfig {
+  nodeId: string;
+  macroId: string;
+  macroData: any;
+}
+
+export type NodeInstance =
+  | RefNodeInstance
+  | InlineNodeInstance
+  | MacroNodeInstance;
+
+export type ResolvedNodeInstance =
+  | RefNodeInstance
+  | ResolvedInlineNodeInstance
+  | ResolvedMacroNodeInstance;
 
 export const nodeInstance = (
   id: string,
@@ -39,20 +70,46 @@ export const inlineNodeInstance = (
   node: Node,
   config?: InputPinsConfig,
   pos?: Pos
-): NodeInstance => ({
-  id,
-  node,
-  inputConfig: config || {},
-  pos: pos || { x: 0, y: 0 },
-});
+): NodeInstance =>
+  ({
+    id,
+    node,
+    inputConfig: config || {},
+    pos: pos || { x: 0, y: 0 },
+  } as InlineNodeInstance);
+
+export const macroNodeInstance = (
+  id: string,
+  macroId: string,
+  macroData: any,
+  config?: InputPinsConfig,
+  pos?: Pos
+): ResolvedMacroNodeInstance =>
+  ({
+    id,
+    macroId,
+    macroData,
+    inputConfig: config || {},
+    nodeId: `${macroId}__${id}`, // TODO: lift this concatenation to a higher level
+    pos: pos || { x: 0, y: 0 },
+  } as ResolvedMacroNodeInstance);
 
 export const isInlineNodeInstance = (
   ins: NodeInstance
 ): ins is InlineNodeInstance => {
   return !!(ins as any).node;
 };
 export const isRefNodeInstance = (ins: NodeInstance): ins is RefNodeInstance =>
-  !isInlineNodeInstance(ins);
+  !!(ins as any).nodeId && !(ins as any).macroId;
+
+export const isMacroNodeInstance = (
+  ins: NodeInstance
+): ins is MacroNodeInstance => !!(ins as any).macroId;
+
+export const isResolvedMacroNodeInstance = (
+  ins: ResolvedNodeInstance | NodeInstance
+): ins is ResolvedMacroNodeInstance =>
+  !!(ins as any).macroId && !!(ins as any).nodeId;
 
 export const NodeInstance = (
   id: string,

core/src/node/node.ts

@@ -11,7 +11,13 @@ import { Subject } from "rxjs";
 
 import { CancelFn, InnerExecuteFn } from "../execute";
 import { ConnectionData } from "../connect";
-import { isInlineNodeInstance, NodeInstance } from "./node-instance";
+import {
+  isInlineNodeInstance,
+  NodeInstance,
+  RefNodeInstance,
+  ResolvedMacroNodeInstance,
+  ResolvedNodeInstance,
+} from "./node-instance";
 import {
   InputPin,
   InputPinMap,
@@ -78,6 +84,11 @@ export interface BaseNode {
    * Node's unique id. {@link VisualNode.instances }  refer use this to refer to the correct node
    */
   id: string;
+
+  /**
+   * A human readable name for the node. Used in the visual editor.
+   */
+  displayName?: string;
   /**
    * Is displayed in the visual editor and used to search for nodes.
    */
@@ -174,6 +185,33 @@ export interface CodeNode extends BaseNode {
   customView?: CustomNodeViewFn;
 }
 
+export interface MacroNode<T> {
+  id: string;
+  displayNameBuilder?: (data: T) => string;
+  defaultStyle?: NodeStyle;
+  description?: string;
+  definitionBuilder: (data: T) => Omit<CodeNodeDefinition, "id">;
+  runFnBuilder: (data: T) => CodeNode["run"];
+  defaultData: T;
+
+  /**
+   * Assumes you are bundling the editor component using webpack library+window config.
+   * The name of the window variable that holds the component should be __MacroNode__{id}
+   * The path should be relative to the root of the project (package.json location)
+   */
+  editorComponentBundlePath: string;
+}
+
+export type MacroNodeDefinition<T> = Omit<
+  MacroNode<T>,
+  "definitionBuilder" | "runFnBuilder" | "editorComponentBundlePath"
+> & {
+  /**
+   * Resolver will use this to load the editor component bundle into the editor
+   */
+  editorComponentBundleContent: string;
+};
+
 export enum InlineValueNodeType {
   VALUE = "value",
   FUNCTION = "function",
@@ -211,6 +249,10 @@ export interface VisualNode extends BaseNode {
   customView?: CustomNodeViewFn;
 }
 
+export interface ResolvedVisualNode extends VisualNode {
+  instances: ResolvedNodeInstance[];
+}
+
 export type Node = CodeNode | CustomNode;
 
 export type ImportableSource = {
@@ -224,6 +266,7 @@ export type CustomNode = VisualNode | InlineValueNode;
 export type CodeNodeDefinition = Omit<CodeNode, "run">;
 
 export type NodeDefinition = CustomNode | CodeNodeDefinition;
+export type NodeOrMacroDefinition = NodeDefinition | MacroNodeDefinition<any>;
 
 export type NodeModuleMetaData = {
   imported?: boolean;
@@ -240,6 +283,20 @@ export const isCodeNode = (p: Node | NodeDefinition | any): p is CodeNode => {
   return isBaseNode(p) && typeof (p as CodeNode).run === "function";
 };
 
+export const isMacroNode = (p: any): p is MacroNode<any> => {
+  return p && typeof (p as MacroNode<any>).runFnBuilder === "function";
+};
+
+export const isMacroNodeDefinition = (
+  p: any
+): p is MacroNodeDefinition<any> => {
+  return (
+    p &&
+    typeof (p as MacroNodeDefinition<any>).editorComponentBundleContent ===
+      "string"
+  );
+};
+
 export const isVisualNode = (p: Node | NodeDefinition): p is VisualNode => {
   return !!(p as VisualNode).instances;
 };
@@ -335,10 +392,18 @@ export const getNode = (
   idOrIns: string | NodeInstance,
   resolvedNodes: NodesCollection
 ): Node => {
-  if (typeof idOrIns !== "string" && isInlineNodeInstance(idOrIns)) {
-    return idOrIns.node;
+  const isOrInsResolved = idOrIns as string | ResolvedNodeInstance; // ugly type hack to avoid fixing the whole Resolved instances cases caused by macros. TODO: fix this by refactoring all places to use "ResolvedNodeInstance"
+  if (
+    typeof isOrInsResolved !== "string" &&
+    isInlineNodeInstance(isOrInsResolved)
+  ) {
+    return isOrInsResolved.node;
   }
-  const id = typeof idOrIns === "string" ? idOrIns : idOrIns.nodeId;
+  const id =
+    typeof isOrInsResolved === "string"
+      ? isOrInsResolved
+      : isOrInsResolved.nodeId;
+
   const node = resolvedNodes[id];
   if (!node) {
     throw new Error(`Node with id ${id} not found`);
@@ -353,7 +418,10 @@ export const getNodeDef = (
   if (typeof idOrIns !== "string" && isInlineNodeInstance(idOrIns)) {
     return idOrIns.node;
   }
-  const id = typeof idOrIns === "string" ? idOrIns : idOrIns.nodeId;
+  const id =
+    typeof idOrIns === "string"
+      ? idOrIns
+      : (idOrIns as RefNodeInstance | ResolvedMacroNodeInstance).nodeId;
   const node = resolvedNodes[id];
   if (!node) {
     console.error(`Node with id ${id} not found`);

dev-server/src/client.ts

@@ -3,6 +3,8 @@ import {
   FlydeFlow,
   ImportableSource,
   ResolvedDependenciesDefinitions,
+  ResolvedFlydeFlow,
+  ResolvedFlydeFlowDefinition,
 } from "@flyde/core";
 import { FolderStructure } from "./fs-helper/shared";
 import type { ImportablesResult } from "./service/scan-importable-nodes";
@@ -25,9 +27,9 @@ export const createDevServerClient = (baseUrl: string) => {
     },
     resolveDefinitions: (
       filename: string
-    ): Promise<ResolvedDependenciesDefinitions> => {
+    ): Promise<ResolvedFlydeFlowDefinition> => {
       return axios
-        .get(`${baseUrl}/resolveDefinitions?filename=${filename}`)
+        .get(`${baseUrl}/resolveFlow?filename=${filename}`)
         .then((res) => res.data);
     },
     getImportables: (filename: string): Promise<ImportablesResult> => {

dev-server/src/server.ts

@@ -4,7 +4,7 @@ import { createService } from "./service/service";
 import { setupRemoteDebuggerServer } from "@flyde/remote-debugger/dist/setup-server";
 import { createServer } from "http";
 import { scanImportableNodes } from "./service/scan-importable-nodes";
-import { deserializeFlow, resolveDependencies } from "@flyde/resolver";
+import { deserializeFlow, resolveFlowByPath } from "@flyde/resolver";
 import { join } from "path";
 
 import resolveFrom = require("resolve-from");
@@ -68,7 +68,7 @@ export const runDevServer = (
     }
   });
 
-  app.get("/resolveDefinitions", async (req, res) => {
+  app.get("/resolveFlow", async (req, res) => {
     try {
       const { filename } = req.query as { filename: string };
       if (!filename) {
@@ -77,9 +77,8 @@ export const runDevServer = (
       }
 
       const fullPath = resolveFrom(rootDir, filename);
-      const flow = deserializeFlow(readFileSync(fullPath, "utf-8"), fullPath);
-      const deps = await resolveDependencies(flow, "definition", fullPath);
-      res.send({ ...deps, [flow.node.id]: flow.node });
+      const resolvedFlow = await resolveFlowByPath(fullPath, "definition");
+      res.send(resolvedFlow);
     } catch (e) {
       console.error(e);
       res.status(400).send(e);

dev-server/src/service/generate-node-from-prompt.ts

@@ -2,10 +2,9 @@ import {
   CodeNode,
   ImportableSource,
   ImportedNode,
-  RunNodeFunction,
   randomInt,
 } from "@flyde/core";
-import { resolveCodeNodeDependencies } from "@flyde/resolver";
+import { resolveCodeNodeDependencies, resolveFlow } from "@flyde/resolver";
 import axios from "axios";
 import { existsSync, writeFileSync } from "fs";
 
@@ -77,7 +76,7 @@ export async function generateAndSaveNode(
   }
 
   const node: ImportedNode = {
-    ...maybeNode.node,
+    ...(maybeNode.node as CodeNode),
     source: { path: filePath, export: maybeNode.exportName },
   };
   return { node, module: `./${fileName}.flyde.ts` };