From c4c745f4f952c6c294613a66442573cd1512309c Mon Sep 17 00:00:00 2001 From: fishtreesugar Date: Fri, 31 Jan 2025 19:29:57 -0600 Subject: [PATCH 1/2] init spark rewriting --- .formatter.exs | 20 +- lib/workflow.ex | 607 +----------------- lib/workflow/dsl.ex | 243 +++++++ .../dsl/transformers/code_generation.ex | 317 +++++++++ lib/workflow/options.ex | 183 ------ mix.exs | 6 +- mix.lock | 2 + test/support/pacer_doc_sample.ex | 7 +- 8 files changed, 602 insertions(+), 783 deletions(-) create mode 100644 lib/workflow/dsl.ex create mode 100644 lib/workflow/dsl/transformers/code_generation.ex delete mode 100644 lib/workflow/options.ex diff --git a/.formatter.exs b/.formatter.exs index d2cda26..2ab8ba2 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,22 @@ # Used by "mix format" +spark_locals_without_parens = [ + batch: 2, + batch: 3, + default: 1, + dependencies: 1, + doc: 1, + field: 1, + field: 2, + guard: 1, + resolver: 1, + virtual?: 1 +] + [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"], + plugins: [Spark.Formatter], + locals_without_parens: spark_locals_without_parens, + export: [ + locals_without_parens: spark_locals_without_parens + ] ] diff --git a/lib/workflow.ex b/lib/workflow.ex index 3b24916..038a35f 100644 --- a/lib/workflow.ex +++ b/lib/workflow.ex @@ -12,7 +12,6 @@ defmodule Pacer.Workflow do the `graph/1` macro, which is explained in more detail in the docs below. Note that when using `Pacer.Workflow`, you can pass the following options: - #{NimbleOptions.docs(Pacer.Workflow.Options.graph_options())} The following is a list of the main ideas and themes underlying `Pacer.Workflow` @@ -372,534 +371,33 @@ defmodule Pacer.Workflow do and the default returned. """ + use Spark.Dsl, default_extensions: [extensions: [Pacer.Workflow.Dsl]] + alias Pacer.Config alias Pacer.Workflow.Error - alias Pacer.Workflow.FieldNotSet - alias Pacer.Workflow.Options require Logger require Pacer.Docs - @example_graph_message """ - Ex.: - - defmodule MyValidGraph do - use Pacer.Workflow - - graph do - field(:custom_field) - field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) - field(:field_with_default, default: "this is a default value") - - batch :http_requests, timeout: :timer.seconds(1) do - field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field], default: 5) - - field(:request_2, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field, :field_a], default: "this is the default for request2") - field(:request_3, resolver: &__MODULE__.do_work/1, default: :this_default) - end - end - - def do_work(_), do: :ok - end - """ - - @default_batch_options Options.default_batch_options() - defmacro __using__(opts) do - quote do - import Pacer.Workflow, - only: [ - graph: 1, - batch: 2, - batch: 3, - field: 1, - field: 2, - find_cycles: 1 - ] - - alias Pacer.Workflow.Options - require Pacer.Docs - - Module.register_attribute(__MODULE__, :pacer_generate_docs?, accumulate: false) - - generate_docs? = Keyword.get(unquote(opts), :generate_docs?, true) - - Module.put_attribute( - __MODULE__, - :pacer_generate_docs?, - generate_docs? - ) - - Module.register_attribute(__MODULE__, :pacer_debug_mode?, accumulate: false) - debug_mode? = Keyword.get(unquote(opts), :debug_mode?, false) - Module.put_attribute(__MODULE__, :pacer_debug_mode?, debug_mode?) - - batch_telemetry_options = Keyword.get(unquote(opts), :batch_telemetry_options, %{}) - - Module.put_attribute(__MODULE__, :pacer_batch_telemetry_options, batch_telemetry_options) - - Module.register_attribute(__MODULE__, :pacer_docs, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_graph_vertices, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_field_to_batch_mapping, accumulate: false) - Module.register_attribute(__MODULE__, :pacer_fields, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_struct_fields, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_dependencies, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_resolvers, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_batches, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_batch_dependencies, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_batch_guard_functions, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_batch_options, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_batch_resolvers, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_vertices, accumulate: true) - Module.register_attribute(__MODULE__, :pacer_virtual_fields, accumulate: true) - end - end - - @doc """ - The graph/1 macro is the main entrypoint into Pacer.Workflow to create a dependency graph struct. - `use` the `Pacer.Workflow` macro at the top of your module and proceed to define your fields and/or batches. - - Example - ```elixir - defmodule MyValidGraph do - use Pacer.Workflow - - graph do - field(:custom_field) - field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) - field(:field_with_default, default: "this is a default value") - - batch :http_requests do - field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field, :field_a]) - field(:request_2, resolver: &__MODULE__.do_work/1) - end - end - - def do_work(_), do: :ok - end - ``` - - Your module may only define ONE graph per module. - - The above example will also create a struct with all of the fields defined within the graph, as follows: - - ```elixir - %MyValidGraph{ - custom_field: nil, - field_a: nil, - field_with_default: "this is a default value", - request_1: nil, - request_2: nil - } - ``` - - - The graph macro gives you access to some defined metadata functions, such as (using the above example graph): - - `MyValidGraph.__graph__(:fields)` - - `MyValidGraph.__graph__(:dependencies, :http_requests)` - - `MyValidGraph.__graph__(:resolver, :field_a)` - - **Caution: These metadata functions are mostly intended for Pacer's internal use. Do not rely on their return values in runtime - code as they may change as changes are made to the interface for Pacer. - """ - # credo:disable-for-lines:79 Credo.Check.Refactor.ABCSize - # credo:disable-for-lines:79 Credo.Check.Refactor.CyclomaticComplexity - defmacro graph(do: block) do - caller = __CALLER__ - - prelude = - quote do - Module.put_attribute(__MODULE__, :pacer_field_to_batch_mapping, %{}) - - if line = Module.get_attribute(__MODULE__, :pacer_graph_defined) do - raise Error, """ - Module #{inspect(__MODULE__)} already defines a graph on line #{line} - """ + {batch_telemetry_options, opts} = + Keyword.pop( + opts, + :batch_telemetry_options, + quote do + %{} end - - @pacer_graph_defined unquote(caller.line) - - @after_compile Pacer.Workflow - - unquote(block) - end - - # credo:disable-for-lines:155 Credo.Check.Refactor.LongQuoteBlocks - postlude = - quote unquote: false do - batched_dependencies = - Enum.reduce(@pacer_batch_dependencies, %{}, fn {batch_name, _field_name, deps}, - batched_dependencies -> - Map.update(batched_dependencies, batch_name, deps, fn existing_val -> - Enum.uniq(Enum.concat(existing_val, deps)) - end) - end) - - batched_field_dependencies = - Enum.reduce(@pacer_batch_dependencies, %{}, fn {_batch_name, field_name, deps}, - batched_field_dependencies -> - Map.put(batched_field_dependencies, field_name, deps) - end) - - batched_fields = - Enum.reduce(@pacer_batch_dependencies, %{}, fn {batch_name, field_name, _deps}, acc -> - Map.update(acc, batch_name, [field_name], fn existing_val -> - [field_name | existing_val] - end) - end) - - batched_resolvers = - Enum.reduce(@pacer_batch_resolvers, %{}, fn {batch, field, resolver}, acc -> - Map.update(acc, batch, [{field, resolver}], fn fields_and_resolvers -> - [{field, resolver} | fields_and_resolvers] - end) - end) - - batch_guard_functions = - Enum.reduce(@pacer_batch_guard_functions, %{}, fn {_batch, field, guard}, acc -> - Map.put(acc, field, guard) - end) - - @pacer_batch_dependencies - |> Enum.reduce([], fn {batch, _, _}, batches -> [batch | batches] end) - |> Enum.each(fn batch -> - fields_for_batch = - batched_fields - |> Map.get(batch, []) - |> MapSet.new() - - deps_for_batch = - batched_dependencies - |> Map.get(batch, []) - |> MapSet.new() - - intersection = MapSet.intersection(fields_for_batch, deps_for_batch) - - unless Enum.empty?(intersection) do - raise Error, - """ - Found at least one invalid field dependency inside of a Pacer.Workflow batch. - Invalid dependencies: #{inspect(Enum.to_list(intersection))} - Graph module: #{inspect(__MODULE__)} - - Fields that are defined within a batch MUST not have dependencies on other - fields in the same batch because their resolvers will run concurrently. - - You may need to rearrange an invalid field (or fields) out of your batch - if the field does have a hard dependency on another field in the batch. - """ - end - end) - - # Instantiate the graph with the list of vertices derived from the graph definition - initial_graph = Graph.add_vertices(Graph.new(), @pacer_graph_vertices) - - # Build the graph edges, where edges go from dependency -> dependent field OR batch. - # We concat together `@pacer_batch_dependencies` with `@pacer_dependencies`: - # - `@pacer_dependencies` is a list of `{field_name, list(field_dependencies)}` - # - `@pacer_batch_dependencies` is a list of 3-tuples: `{batch_name, field_name, list(field_dependencies)}`. - # The dependencies of all fields defined within a batch bubble up to the batch - # itself. - # Batch names become vertices in the graph, but the fields under - # a batch are not represented in the graph as vertices: they collapse into the - # vertex for the batch. This means that there is special treatment for batches. - # Fields not within a batch ARE represented as vertices in the graph. So building - # edges for fields not within a batch is relatively straighforward: take each dependency - # the field defines and draw an edge from the dependency to the field. - # Edges for batches require that we concat the dependencies for all fields within a batch, - # then take that list of dependencies and iterate of each dependency. The edges then - # become `dependency -> batch`. - # Finally, if a dependency is itself part of a batch, we have to lookup the - # batch it belongs to. Then, instead of drawing an edge from the dependency itself - # to the field or batch that depends on it, we draw an edge from the dependency's batch - # to the field or batch that depends on it. - # The `@pacer_field_to_batch_mapping` is a map with plain fields as keys and the batch they - # belong to as values. If a field does not belong to a batch, a lookup into the `@pacer_field_to_batch_mapping` - # will return `nil`. We use this in the case statements within the `Enum.flat_map/2` call - # to indicate whether or not we need to replace a field with the batch it belongs to when - # drawing the edges. - graph_edges = - @pacer_batch_dependencies - |> Enum.concat(@pacer_dependencies) - |> Enum.flat_map(fn - {batch, field, deps} -> - for dep <- deps do - case Map.get(@pacer_field_to_batch_mapping, dep) do - nil -> Graph.Edge.new(dep, batch) - dependency_batch -> Graph.Edge.new(dependency_batch, batch) - end - end - - {field, deps} -> - for dep <- deps do - case Map.get(@pacer_field_to_batch_mapping, dep) do - nil -> Graph.Edge.new(dep, field) - batch -> Graph.Edge.new(batch, field) - end - end - end) - - # Update the graph with the graph edges - graph = Graph.add_edges(initial_graph, graph_edges) - - visualization = Graph.to_dot(graph) - # Technically, the call to `topsort/1` will fail (return `false`) if - # there are any cycles in the graph, so we can indirectly derive whether or not - # we have any cycles based on the return value from the call to `topsort/1` below. - # However, we want to raise and show an error message that explicitly indicates - # what vertices of the graph form the cycle so the user can more easily find and - # fix any cycles they may have introduced. - _ = find_cycles(graph) - - topsort = Graph.topsort(graph) - - if @pacer_generate_docs? do - Pacer.Docs.generate() - end - - # The call to `topsort/1` above returns all vertices in the graph. However, - # not every vertex in the graph is going to have work to do, which we are deriving - # based on whether or not the vertex has an associated resolver (for a field) or - # list of resolvers (for batches). Any vertex that has NO associated resolvers - # gets filtered out. - # The result of this is what gets returned from the generated `def __graph__(:evaluation_order)` - # function definition below. We will use this to iterate through the resolvers and execute them - # in the order returned by the topological_sort. - vertices_with_work_to_do = - Enum.filter(topsort, fn vertex -> - Keyword.get(@pacer_resolvers, vertex) || Map.get(batched_resolvers, vertex) - end) - - defstruct Enum.reverse(@pacer_struct_fields) - - def __config__(:batch_telemetry_options), do: @pacer_batch_telemetry_options - def __config__(:debug_mode?), do: @pacer_debug_mode? - def __config__(_), do: nil - - def __graph__(:fields), do: Enum.reverse(@pacer_fields) - def __graph__(:dependencies), do: Enum.reverse(@pacer_dependencies) - def __graph__(:batch_dependencies), do: Enum.reverse(@pacer_batch_dependencies) - def __graph__(:evaluation_order), do: unquote(vertices_with_work_to_do) - def __graph__(:virtual_fields), do: Enum.reverse(@pacer_virtual_fields) - def __graph__(:visualization), do: unquote(visualization) - - for {name, deps} <- @pacer_dependencies do - def __graph__(:dependencies, unquote(name)), do: unquote(deps) - end - - for {name, deps} <- batched_field_dependencies do - def __graph__(:batched_field_dependencies, unquote(name)), do: unquote(deps) - end - - for {name, guard} <- batch_guard_functions do - def __graph__(:batched_field_guard_functions, unquote(name)), do: unquote(guard) - end - - def __graph__(:batched_field_guard_functions, _field), do: nil - - for {batch_name, batch_options} <- @pacer_batch_options do - def __graph__(unquote(batch_name), :options), do: unquote(batch_options) - end - - for {batch_name, deps} <- batched_dependencies do - def __graph__(:dependencies, unquote(batch_name)), do: unquote(deps) - end - - for {batch_name, fields} <- batched_fields do - def __graph__(:batch_fields, unquote(batch_name)), do: unquote(fields) - end - - for {name, resolver} <- @pacer_resolvers do - def __graph__(:resolver, unquote(name)), do: {:field, unquote(resolver)} - end - - for {batch_name, fields_and_resolvers} <- batched_resolvers do - def __graph__(:resolver, unquote(batch_name)), - do: {:batch, unquote(fields_and_resolvers)} - end - end - - quote do - unquote(prelude) - unquote(postlude) - end - end - - @doc """ - The batch/3 macro is to be invoked when grouping fields with resolvers that will run in parallel. - - Reminder: - - The batch must be named and unique. - - The fields within the batch must not have dependencies on one another since they will run concurrently. - - The fields within the batch must each declare a resolver function. - - __NOTE__: In general, only batch fields whose resolvers contain potentially high-latency operations, such as network calls. - - Example - ```elixir - defmodule MyValidGraph do - use Pacer.Workflow - - graph do - field(:custom_field) - - batch :http_requests do - field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) - field(:request_2, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) - field(:request_3, resolver: &__MODULE__.do_work/1) - end - end - - def do_work(_), do: :ok - end - ``` - - Field options for fields defined within a batch have one minor requirement difference - from fields not defined within a batch: batched fields MUST always define a resolver function, - regardless of whether or not they define any dependencies. - - ## Batch Field Options - #{NimbleOptions.docs(Options.batch_fields_definition())} - - ## Batch options - #{NimbleOptions.docs(Options.batch_definition())} - """ - defmacro batch(name, options \\ @default_batch_options, do: block) do - prelude = - quote do - Module.put_attribute( - __MODULE__, - :pacer_batch_options, - {unquote(name), - unquote(options) - |> Keyword.put(:on_timeout, :kill_task) - |> Pacer.Workflow.validate_options(Options.batch_definition())} - ) - - if unquote(name) in Module.get_attribute(__MODULE__, :pacer_batches) do - raise Error, """ - Found duplicate batch name `#{unquote(name)}` in graph module #{inspect(__MODULE__)}. - Batch names within a single graph instance must be unique. - """ - else - Module.put_attribute(__MODULE__, :pacer_batches, unquote(name)) - end - end - - postlude = - Macro.postwalk(block, fn ast -> - case ast do - {:field, metadata, [field_name | args]} -> - batched_ast = - quote do - {unquote(:batch), unquote(name), unquote(field_name)} - end - - {:field, metadata, [batched_ast | args]} - - _ -> - ast - end - end) - - quote do - unquote(prelude) - unquote(postlude) - end - end - - @doc """ - The field/2 macro maps fields one-to-one to keys on the struct created via the graph definition. - - Fields must be unique within a graph instance. - - ## Options: - There are specific options that are allowed to be passed in to the field macro, as indicated below: - - #{NimbleOptions.docs(Pacer.Workflow.Options.fields_definition())} - """ - defmacro field(name, options \\ []) do - quote do - Pacer.Workflow.__field__(__MODULE__, unquote(name), unquote(options)) - end - end - - @spec __field__(module(), atom(), Keyword.t()) :: :ok | no_return() - def __field__(module, {:batch, batch_name, field_name}, options) do - opts = validate_options(options, Options.batch_fields_definition()) - - _ = - module - |> Module.get_attribute(:pacer_field_to_batch_mapping) - |> Map.put(field_name, batch_name) - |> tap(fn mapping -> - Module.put_attribute(module, :pacer_field_to_batch_mapping, mapping) - end) - - Module.put_attribute( - module, - :pacer_batch_resolvers, - {batch_name, field_name, Keyword.fetch!(opts, :resolver)} - ) - - Module.put_attribute( - module, - :pacer_docs, - {field_name, batch_name, Keyword.get(opts, :doc, "")} - ) - - Module.put_attribute( - module, - :pacer_batch_dependencies, - {batch_name, field_name, Keyword.get(opts, :dependencies, [])} - ) - - if Keyword.get(opts, :guard) && is_function(Keyword.get(opts, :guard), 1) do - Module.put_attribute( - module, - :pacer_batch_guard_functions, - {batch_name, field_name, Keyword.fetch!(opts, :guard)} ) - end - - put_field_attributes(module, field_name, opts) - - unless Enum.member?(Module.get_attribute(module, :pacer_graph_vertices), batch_name) do - Module.put_attribute(module, :pacer_graph_vertices, batch_name) - end - end - def __field__(module, name, options) do - opts = validate_options(options, Options.fields_definition()) + {debug_mode?, opts} = Keyword.pop(opts, :debug_mode?) - deps = Keyword.fetch!(opts, :dependencies) - Module.put_attribute(module, :pacer_dependencies, {name, deps}) - - if Keyword.get(opts, :virtual?) do - Module.put_attribute(module, :pacer_virtual_fields, name) - end - - Module.put_attribute(module, :pacer_docs, {name, Keyword.get(opts, :doc, "")}) + quote do + def __config__(:batch_telemetry_options), do: unquote(batch_telemetry_options) + def __config__(:debug_mode?), do: unquote(debug_mode?) + def __config__(_), do: nil - unless deps == [] do - register_and_validate_resolver(module, name, opts) + unquote(super(opts)) end - - put_field_attributes(module, name, opts) - - Module.put_attribute(module, :pacer_graph_vertices, name) - end - - defp put_field_attributes(module, field_name, opts) do - :ok = ensure_no_duplicate_fields(module, field_name) - - Module.put_attribute( - module, - :pacer_struct_fields, - {field_name, Keyword.get(opts, :default, %FieldNotSet{})} - ) - - Module.put_attribute(module, :pacer_fields, field_name) end @doc """ @@ -934,83 +432,6 @@ defmodule Pacer.Workflow do """ end - @spec __after_compile__(Macro.Env.t(), binary()) :: :ok | no_return() - def __after_compile__(%{module: module} = _env, _) do - _ = validate_dependencies(module) - - :ok - end - - @spec ensure_no_duplicate_fields(module(), atom()) :: :ok | no_return() - defp ensure_no_duplicate_fields(module, field_name) do - if Enum.member?(Module.get_attribute(module, :pacer_fields), field_name) do - raise Error, "Found duplicate field in graph instance for #{inspect(module)}: #{field_name}" - end - - :ok - end - - @spec register_and_validate_resolver(module(), atom(), Keyword.t()) :: :ok | no_return() - defp register_and_validate_resolver(module, name, options) do - case Keyword.fetch(options, :resolver) do - {:ok, resolver} -> - Module.put_attribute(module, :pacer_resolvers, {name, resolver}) - :ok - - :error -> - error_message = """ - Field #{name} in #{inspect(module)} declared at least one dependency, but did not specify a resolver function. - Any field that declares at least one dependency must also declare a resolver function. - - #{@example_graph_message} - """ - - raise Error, error_message - end - end - - @spec validate_options(Keyword.t(), NimbleOptions.t()) :: Keyword.t() - def validate_options(options, schema) do - case NimbleOptions.validate(options, schema) do - {:ok, options} -> - options - - {:error, %NimbleOptions.ValidationError{} = validation_error} -> - raise Error, """ - #{Exception.message(validation_error)} - - #{@example_graph_message} - """ - end - end - - @spec validate_dependencies(module()) :: :ok | no_return() - def validate_dependencies(module) do - deps_set = - :dependencies - |> module.__graph__() - |> Enum.concat(module.__graph__(:batch_dependencies)) - |> Enum.flat_map(fn - {_, deps} -> deps - {_, _, deps} -> deps - end) - |> MapSet.new() - - field_set = MapSet.new(module.__graph__(:fields)) - - unless MapSet.subset?(deps_set, field_set) do - invalid_deps = MapSet.difference(deps_set, field_set) - - raise Error, - """ - Found at least one invalid dependency in graph definiton for #{inspect(module)} - Invalid dependencies: #{inspect(Enum.to_list(invalid_deps))} - """ - end - - :ok - end - @doc """ Takes a struct that has been defined via the `Pacer.Workflow.graph/1` macro. `execute` will run/execute all of the resolvers defined in the definition of the diff --git a/lib/workflow/dsl.ex b/lib/workflow/dsl.ex new file mode 100644 index 0000000..5b36efc --- /dev/null +++ b/lib/workflow/dsl.ex @@ -0,0 +1,243 @@ +defmodule Pacer.Workflow.Dsl do + defmodule Field do + defstruct [:name, :default, :dependencies, :resolver, :virtual?, :doc] + end + + @field_schema [ + name: [ + type: :atom, + required: true, + doc: "field name" + ], + dependencies: [ + default: [], + type: {:list, :atom}, + doc: """ + A list of dependencies from the graph. + Dependencies are specified as atoms, and each dependency must + be another field in the same graph. + + Remember that cyclical dependencies are strictly not allowed, + so fields cannot declare dependencies on themselves nor on + any other field that has already declared a dependency on the + current field. + + If the `dependencies` option is not given, it defaults to an + empty list, indicating that the field has no dependencies. + This will be the case if the field is a constant or can be + constructed from values already available in the environment. + """ + ], + doc: [ + type: :string, + required: false, + default: "", + doc: """ + Allows users to document the field and provide background and/or context + on what the field is intended to be used for, what kind of data the field + contains, and how the data for the field is constructed. + """ + ], + resolver: [ + type: {:fun, 1}, + required: false, + doc: """ + A resolver is a 1-arity function that specifies how to calculate + the value for a field. + + The argument passed to the function will be a map that contains + all of the field's declared dependencies. + + For example, if we have a field like this: + + ```elixir + field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) + ``` + + The resolver `RequestHandler.resolve/1` would be passed a map that looks like this: + ```elixir + %{api_key: "", url: "https://some.endpoint.com"} + ``` + + If the field has no dependencies, the resolver will receive an empty map. Note though + that resolvers are only required for fields with no dependencies if the field is inside + of a batch. If your field has no dependencies and is not inside a batch, you can skip + defining a resolver and initialize your graph struct with a value that is either constant + or readily available where you are constructing the struct. + + The result of the resolver will be placed on the graph struct under the field's key. + + For the above, assuming a graph that looks like this: + + ```elixir + defmodule MyGraph do + use Pacer.Workflow + + graph do + field(:api_key) + field(:url) + field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) + end + end + ``` + + Then when the `RequestHandler.resolve/1` runs an returns a value of, let's say, `%{response: "important response"}`, + your graph struct would look like this: + + ```elixir + %MyGraph{ + api_key: "", + url: "https://some.endpoint.com", + request: %{response: "important response"} + } + ``` + """ + ], + default: [ + type: :any, + default: + quote do + %Pacer.Workflow.FieldNotSet{} + end, + doc: """ + The default value for the field. If no default is given, the default value becomes + `#Pacer.Workflow.FieldNotSet<>`. + """ + ], + virtual?: [ + default: false, + type: :boolean, + doc: """ + A virtual field is used for intermediate or transient computation steps during the workflow and becomes a + node in the workflow's graph, but does not get returned in the results of the workflow execution. + + In other words, virtual keys will not be included in the map returned by calling `Pacer.Workflow.execute/1`. + + The intent of a virtual field is to allow a spot for intermediate and/or transient calculation steps but + to avoid the extra memory overhead that would be associated with carrying these values downstream if, for example, + the map returned from `Pacer.Workflow.execute/1` is stored in a long-lived process state; intermediate or transient + values can cause unnecessary memory bloat if they are carried into process state where they are not neeeded. + """ + ] + ] + + @field %Spark.Dsl.Entity{ + name: :field, + args: [:name], + target: Field, + schema: @field_schema, + transform: {__MODULE__, :resolver_with_dep_graph, []} + } + + defmodule BatchField do + defstruct [:name, :default, :dependencies, :resolver, :virtual?, :guard, :doc] + end + + @batch_field_schema Keyword.merge( + @field_schema, + default: [ + required: true, + type: :any, + doc: """ + The default value for the field. If no default is given, the default value becomes + `#Pacer.Workflow.FieldNotSet<>`. + """ + ], + resolver: [ + required: true, + type: {:fun, 1}, + doc: """ + A guard is a 1-arity function that takes in a map with the field's dependencies and returns either true or false. + If the function returns `false`, it means there is no work to do and thus no reason to spin up another process + to run the resolver function. In this case, the field's default value is returned. + If the function returns `true`, the field's resolver will run in a separate process. + """ + ], + guard: [ + type: {:fun, 1}, + doc: """ + A guard is a 1-arity function that takes in a map with the field's dependencies and returns either true or false. + If the function returns `false`, it means there is no work to do and thus no reason to spin up another process + to run the resolver function. In this case, the field's default value is returned. + If the function returns `true`, the field's resolver will run in a separate process. + """ + ] + ) + + @batch_field %Spark.Dsl.Entity{ + name: :field, + args: [:name], + target: BatchField, + schema: @batch_field_schema, + transform: {__MODULE__, :resolver_with_dep_graph, []} + } + + def resolver_with_dep_graph(field_entity) do + if field_entity.dependencies != [] and is_nil(field_entity.resolver) do + {:error, + """ + Field #{field_entity.name} declared at least one dependency, but did not specify a resolver function. + Any field that declares at least one dependency must also declare a resolver function. + """} + else + {:ok, field_entity} + end + end + + defmodule Batch do + defstruct [:name, :fields, :batch_options] + end + + @batch_schema [ + name: [ + type: :atom, + required: true + ], + batch_options: [ + type: + {:keyword_list, + timeout: [ + type: :non_neg_integer, + required: false, + default: 1000, + doc: """ + The task that is timed out is killed and returns {:exit, :timeout}. + This :kill_task option only exits the task process that fails and not the process that spawned the task. + """ + ], + on_timeout: [ + type: :atom, + required: false, + default: :kill_task, + doc: """ + The time in milliseconds that the batch is allowed to run for. + Defaults to 1,000 (1 second). + """ + ]}, + default: [on_timeout: :kill_task, timeout: 1000] + ] + ] + + @batch %Spark.Dsl.Entity{ + name: :batch, + args: [:name, :batch_options], + target: Batch, + schema: @batch_schema, + entities: [fields: [@batch_field]] + } + + @graph %Spark.Dsl.Section{ + name: :graph, + schema: [], + entities: [@field, @batch] + } + + use Spark.Dsl.Extension, + sections: [@graph], + transformers: [Pacer.Workflow.Dsl.Transformers.CodeGeneration], + verifiers: [] + + def field_schema, do: @field_schema + def batch_field_schema, do: @batch_field_schema + def batch_schema, do: @batch_schema +end diff --git a/lib/workflow/dsl/transformers/code_generation.ex b/lib/workflow/dsl/transformers/code_generation.ex new file mode 100644 index 0000000..676adf6 --- /dev/null +++ b/lib/workflow/dsl/transformers/code_generation.ex @@ -0,0 +1,317 @@ +defmodule Pacer.Workflow.Dsl.Transformers.CodeGeneration do + use Spark.Dsl.Transformer + + @init_info %{ + pacer_graph_vertices: [], + pacer_field_to_batch_mapping: %{}, + pacer_docs: [], + pacer_batches: [], + pacer_batch_dependencies: [], + pacer_batch_resolvers: [], + pacer_batch_guard_functions: [], + pacer_batch_options: [], + pacer_struct_fields: [], + pacer_dependencies: [], + pacer_virtual_fields: [], + pacer_resolvers: [] + } + + defp collect_field_info(field, info, batch \\ nil) do + info = + info + |> update_in([:pacer_dependencies], fn v -> + if(is_nil(batch), do: [{field.name, field.dependencies} | v], else: v) + end) + |> update_in([:pacer_docs], &[{field.name, field.doc} | &1]) + |> update_in([:pacer_resolvers], &[{field.name, field.resolver} | &1]) + |> update_in([:pacer_struct_fields], &[{field.name, field.default} | &1]) + |> update_in([:pacer_fields], &[field.name | &1]) + |> update_in([:pacer_graph_vertices], fn v -> + if is_nil(batch), do: [field.name | v], else: v + end) + |> update_in([:pacer_virtual_fields], fn v -> + if field.virtual?, do: [field.name | v], else: v + end) + + if not is_nil(batch) do + info + |> update_in([:pacer_field_to_batch_mapping, field.name], fn _ -> batch.name end) + |> update_in( + [:pacer_batch_resolvers], + &[{batch.name, field.name, field.resolver} | &1] + ) + |> update_in( + [:pacer_batch_dependencies], + &[{batch.name, field.name, field.dependencies} | &1] + ) + |> update_in([:pacer_batch_guard_functions], fn v -> + if is_function(field.guard, 1), do: [{field.name, field.guard} | v], else: v + end) + else + info + end + end + + defp collect_batch_info(batch, info) do + info + |> update_in([:pacer_batches], &[batch.name | &1]) + |> update_in([:pacer_batch_options], &[{batch.name, batch.batch_options} | &1]) + |> update_in([:pacer_graph_vertices], fn v -> + if batch.name not in v, do: [batch.name | v], else: v + end) + end + + def transform(dsl_state) do + entities = Spark.Dsl.Transformer.get_entities(dsl_state, [:graph]) + + info = + for entity <- entities, reduce: @init_info do + info -> + case entity do + %Pacer.Workflow.Dsl.Field{} = field -> + collect_field_info(field, info) + + %Pacer.Workflow.Dsl.Batch{} = batch -> + for field <- batch.fields, reduce: collect_batch_info(batch, info) do + info -> collect_field_info(field, info, batch) + end + end + end + + # Instantiate the graph with the list of vertices derived from the graph definition + initial_graph = Graph.add_vertices(Graph.new(), info.pacer_graph_vertices) + + graph_edges = + info.pacer_batch_dependencies + |> Enum.concat(info.pacer_dependencies) + |> Enum.flat_map(fn + {batch, _field, deps} -> + for dep <- deps do + case Map.get(info.pacer_field_to_batch_mapping, dep) do + nil -> Graph.Edge.new(dep, batch) + dependency_batch -> Graph.Edge.new(dependency_batch, batch) + end + end + + {field, deps} -> + for dep <- deps do + case Map.get(info.pacer_field_to_batch_mapping, dep) do + nil -> Graph.Edge.new(dep, field) + batch -> Graph.Edge.new(batch, field) + end + end + end) + + graph = Graph.add_edges(initial_graph, graph_edges) + _ = Pacer.Workflow.find_cycles(graph) + + visualization = Graph.to_dot(graph) + + topsort = Graph.topsort(graph) + + batched_dependencies = + Enum.reduce(info.pacer_batch_dependencies, %{}, fn {batch_name, _field_name, deps}, + batched_dependencies -> + Map.update(batched_dependencies, batch_name, deps, fn existing_val -> + Enum.uniq(Enum.concat(existing_val, deps)) + end) + end) + + batched_field_dependencies = + Enum.reduce(info.pacer_batch_dependencies, %{}, fn {_batch_name, field_name, deps}, + batched_field_dependencies -> + Map.put(batched_field_dependencies, field_name, deps) + end) + + batched_fields = + Enum.reduce(info.pacer_batch_dependencies, %{}, fn {batch_name, field_name, _deps}, acc -> + Map.update(acc, batch_name, [field_name], fn existing_val -> + [field_name | existing_val] + end) + end) + + batched_resolvers = + Enum.reduce(info.pacer_batch_resolvers, %{}, fn {batch, field, resolver}, acc -> + Map.update(acc, batch, [{field, resolver}], fn fields_and_resolvers -> + [{field, resolver} | fields_and_resolvers] + end) + end) + + vertices_with_work_to_do = + Enum.filter(topsort, fn vertex -> + Keyword.get(info.pacer_resolvers, vertex) || Map.get(batched_resolvers, vertex) + end) + + defstruct_ast = + quote do + defstruct unquote(Enum.reverse(info.pacer_struct_fields)) + end + + graph_fun_dependencies_ast = + for {name, deps} <- info.pacer_dependencies do + quote do + def __graph__(:dependencies, unquote(name)), do: unquote(deps) + end + end + + graph_fun_batched_field_dependencies_ast = + for {name, deps} <- batched_field_dependencies do + quote do + def __graph__(:batched_field_dependencies, unquote(name)), do: unquote(deps) + end + end + + graph_fun_batch_guard_functions_ast = + for {name, guard} <- info.pacer_batch_guard_functions do + quote do + def __graph__(:batched_field_guard_functions, unquote(name)), do: unquote(guard) + end + end + + graph_fun_batch_options_ast = + for {batch_name, batch_options} <- info.pacer_batch_options do + quote do + def __graph__(unquote(batch_name), :options), do: unquote(batch_options) + end + end + + graph_fun_batched_dependencies_ast = + for {batch_name, batch_options} <- batched_dependencies do + quote do + def __graph__(:dependencies, unquote(batch_name)), do: unquote(batch_options) + end + end + + graph_fun_batched_fields_ast = + for {batch_name, fields} <- batched_fields do + quote do + def __graph__(:batch_fields, unquote(batch_name)), do: unquote(fields) + end + end + + graph_fun_pacer_resolvers_ast = + for {name, resolver} <- info.pacer_resolvers do + quote do + def __graph__(:resolver, unquote(name)), do: {:field, unquote(resolver)} + end + end + + graph_fun_fields_and_resolvers_ast = + for {batch_name, fields_and_resolvers} <- batched_resolvers do + quote do + def __graph__(:resolver, unquote(batch_name)), + do: {:batch, unquote(fields_and_resolvers)} + end + end + + graph_fun_ast = + quote do + def __graph__(:fields), do: unquote(Enum.reverse(Keyword.keys(info.pacer_struct_fields))) + def __graph__(:dependencies), do: unquote(Enum.reverse(info.pacer_dependencies)) + + def __graph__(:batch_dependencies) do + unquote(Macro.escape(Enum.reverse(info.pacer_batch_dependencies))) + end + + def __graph__(:evaluation_order), do: unquote(vertices_with_work_to_do) + def __graph__(:virtual_fields), do: unquote(Enum.reverse(info.pacer_virtual_fields)) + def __graph__(:visualization), do: unquote(visualization) + + unquote_splicing(graph_fun_dependencies_ast) + unquote_splicing(graph_fun_batched_field_dependencies_ast) + unquote_splicing(graph_fun_batch_guard_functions_ast) + def __graph__(:batched_field_guard_functions, _field), do: nil + unquote_splicing(graph_fun_batch_options_ast) + unquote_splicing(graph_fun_batched_dependencies_ast) + unquote_splicing(graph_fun_batched_fields_ast) + unquote_splicing(graph_fun_pacer_resolvers_ast) + unquote_splicing(graph_fun_fields_and_resolvers_ast) + end + + ast = + quote do + unquote(defstruct_ast) + unquote(graph_fun_ast) + end + + duplicated_field = duplicated_element(info.pacer_struct_fields, &elem(&1, 0)) + duplicated_batch = duplicated_element(info.pacer_batches, &Function.identity/1) + + invalid_dependencies = + invalid_dependencies( + info.pacer_struct_fields, + info.pacer_dependencies, + info.pacer_batch_dependencies + ) + + module = Spark.Dsl.Verifier.get_persisted(dsl_state, :module) + + # Transformers is run before verifiers, if here puts checkings to verifier, + # it will still generate code and cause some warning, e.g. duplicate key `:key` found in struct. + cond do + not is_nil(duplicated_field) -> + {:error, + Spark.Error.DslError.exception( + message: + "Found duplicate field in graph instance for #{inspect(module)}: #{duplicated_field}", + path: [:graph, duplicated_field], + module: module + )} + + not is_nil(duplicated_batch) -> + {:error, + Spark.Error.DslError.exception( + message: """ + Found duplicated batch name `#{duplicated_batch}` in graph module #{inspect(module)}. + Batch names within a single graph instance must be unique. + """, + path: [:graph, :batch, duplicated_batch], + module: module + )} + + not Enum.empty?(invalid_dependencies) -> + {:error, + Spark.Error.DslError.exception( + message: """ + Found at least one invalid dependency in graph definiton for #{inspect(module)} + Invalid dependencies: #{inspect(invalid_dependencies)} + """, + path: [:graph], + module: module + )} + + true -> + {:ok, Spark.Dsl.Transformer.eval(dsl_state, [], ast)} + end + rescue + e in Pacer.Workflow.Error -> + {:error, + Spark.Error.DslError.exception( + message: e.message, + path: [:graph], + module: Spark.Dsl.Verifier.get_persisted(dsl_state, :module) + )} + end + + defp duplicated_element(kw, getter) do + kw + |> Enum.frequencies_by(&getter.(&1)) + |> Stream.filter(fn {_, freq} -> freq > 1 end) + |> Enum.take(1) + |> case do + [] -> nil + [{k, _}] -> k + end + end + + defp invalid_dependencies(struct_fields, field_deps, batch_deps) do + struct_field_names = Keyword.keys(struct_fields) + + Enum.filter( + Enum.flat_map(field_deps, fn {_, deps} -> deps end) ++ + Enum.flat_map(batch_deps, fn {_, _, deps} -> deps end), + fn dep -> dep not in struct_field_names end + ) + |> Enum.uniq() + end +end diff --git a/lib/workflow/options.ex b/lib/workflow/options.ex deleted file mode 100644 index 61ddb8a..0000000 --- a/lib/workflow/options.ex +++ /dev/null @@ -1,183 +0,0 @@ -defmodule Pacer.Workflow.Options do - @moduledoc false - - fields_schema = [ - dependencies: [ - default: [], - type: {:list, :atom}, - doc: """ - A list of dependencies from the graph. - Dependencies are specified as atoms, and each dependency must - be another field in the same graph. - - Remember that cyclical dependencies are strictly not allowed, - so fields cannot declare dependencies on themselves nor on - any other field that has already declared a dependency on the - current field. - - If the `dependencies` option is not given, it defaults to an - empty list, indicating that the field has no dependencies. - This will be the case if the field is a constant or can be - constructed from values already available in the environment. - """ - ], - doc: [ - required: false, - type: :string, - doc: """ - Allows users to document the field and provide background and/or context - on what the field is intended to be used for, what kind of data the field - contains, and how the data for the field is constructed. - """ - ], - resolver: [ - type: {:fun, 1}, - doc: """ - A resolver is a 1-arity function that specifies how to calculate - the value for a field. - - The argument passed to the function will be a map that contains - all of the field's declared dependencies. - - For example, if we have a field like this: - - ```elixir - field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) - ``` - - The resolver `RequestHandler.resolve/1` would be passed a map that looks like this: - ```elixir - %{api_key: "", url: "https://some.endpoint.com"} - ``` - - If the field has no dependencies, the resolver will receive an empty map. Note though - that resolvers are only required for fields with no dependencies if the field is inside - of a batch. If your field has no dependencies and is not inside a batch, you can skip - defining a resolver and initialize your graph struct with a value that is either constant - or readily available where you are constructing the struct. - - The result of the resolver will be placed on the graph struct under the field's key. - - For the above, assuming a graph that looks like this: - - ```elixir - defmodule MyGraph do - use Pacer.Workflow - - graph do - field(:api_key) - field(:url) - field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) - end - end - ``` - - Then when the `RequestHandler.resolve/1` runs an returns a value of, let's say, `%{response: "important response"}`, - your graph struct would look like this: - - ```elixir - %MyGraph{ - api_key: "", - url: "https://some.endpoint.com", - request: %{response: "important response"} - } - ``` - """ - ], - default: [ - type: :any, - doc: """ - The default value for the field. If no default is given, the default value becomes - `#Pacer.Workflow.FieldNotSet<>`. - """ - ], - virtual?: [ - default: false, - type: :boolean, - doc: """ - A virtual field is used for intermediate or transient computation steps during the workflow and becomes a - node in the workflow's graph, but does not get returned in the results of the workflow execution. - - In other words, virtual keys will not be included in the map returned by calling `Pacer.Workflow.execute/1`. - - The intent of a virtual field is to allow a spot for intermediate and/or transient calculation steps but - to avoid the extra memory overhead that would be associated with carrying these values downstream if, for example, - the map returned from `Pacer.Workflow.execute/1` is stored in a long-lived process state; intermediate or transient - values can cause unnecessary memory bloat if they are carried into process state where they are not neeeded. - """ - ] - ] - - @default_batch_timeout :timer.seconds(1) - @default_batch_options [timeout: @default_batch_timeout] - - batch_options_schema = [ - on_timeout: [ - default: :kill_task, - type: :atom, - required: true, - doc: """ - The task that is timed out is killed and returns {:exit, :timeout}. - This :kill_task option only exits the task process that fails and not the process that spawned the task. - """ - ], - timeout: [ - default: @default_batch_timeout, - type: :non_neg_integer, - required: true, - doc: """ - The time in milliseconds that the batch is allowed to run for. - Defaults to 1,000 (1 second). - """ - ] - ] - - batch_fields_schema = - fields_schema - |> Keyword.update!(:resolver, &Keyword.put(&1, :required, true)) - |> Keyword.update!(:default, &Keyword.put(&1, :required, true)) - |> Keyword.put(:guard, - type: {:fun, 1}, - required: false, - doc: """ - A guard is a 1-arity function that takes in a map with the field's dependencies and returns either true or false. - If the function returns `false`, it means there is no work to do and thus no reason to spin up another process - to run the resolver function. In this case, the field's default value is returned. - If the function returns `true`, the field's resolver will run in a separate process. - """ - ) - - graph_schema = [ - generate_docs?: [ - required: false, - type: :boolean, - default: true, - doc: """ - By invoking `use Pacer.Workflow`, Pacer will automatically generate module documentation for you. It will create a section - titled `Pacer Fields` in your moduledoc, either by creating a moduledoc for you dynamically or appending this section to - any existing module documentation you have already provided. - - To opt out of this feature, when you `use Pacer.Workflow`, set this option to false. - """ - ] - ] - - @fields_schema NimbleOptions.new!(fields_schema) - @batch_fields_schema NimbleOptions.new!(batch_fields_schema) - @batch_schema NimbleOptions.new!(batch_options_schema) - @graph_schema NimbleOptions.new!(graph_schema) - - @spec fields_definition() :: NimbleOptions.t() - def fields_definition, do: @fields_schema - - @spec batch_fields_definition() :: NimbleOptions.t() - def batch_fields_definition, do: @batch_fields_schema - - @spec batch_definition() :: NimbleOptions.t() - def batch_definition, do: @batch_schema - - @spec graph_options() :: NimbleOptions.t() - def graph_options, do: @graph_schema - - def default_batch_options, do: @default_batch_options -end diff --git a/mix.exs b/mix.exs index 272ea44..e73ca94 100644 --- a/mix.exs +++ b/mix.exs @@ -63,8 +63,10 @@ defmodule Pacer.MixProject do [ {:ex_doc, "~> 0.34", only: :dev, runtime: false}, {:libgraph, "~> 0.16.0"}, - {:nimble_options, ">= 0.0.0"}, - {:telemetry, "~> 1.2 or ~> 0.4"} + {:spark, "~> 2.2"}, + {:telemetry, "~> 1.2 or ~> 0.4"}, + # for generating `.formatter.exs`'s `spark_locals_without_parens` + {:sourceror, "~> 1.7", only: [:dev, :test]} ] end end diff --git a/mix.lock b/mix.lock index 667fcdc..f0206fe 100644 --- a/mix.lock +++ b/mix.lock @@ -7,5 +7,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "sourceror": {:hex, :sourceror, "1.7.1", "599d78f4cc2be7d55c9c4fd0a8d772fd0478e3a50e726697c20d13d02aa056d4", [:mix], [], "hexpm", "cd6f268fe29fa00afbc535e215158680a0662b357dc784646d7dff28ac65a0fc"}, + "spark": {:hex, :spark, "2.2.45", "19e3a879e80d02853ded85ed7b4c0a84a5d2e395f9d0c884e1a13afbe026929d", [:mix], [{:igniter, ">= 0.3.64 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:sourceror, "~> 1.2", [hex: :sourceror, repo: "hexpm", optional: true]}], "hexpm", "70b272d0ee16e3c10a4f8cf0ef6152840828152e68f2f8e3046e89567f2b49ad"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, } diff --git a/test/support/pacer_doc_sample.ex b/test/support/pacer_doc_sample.ex index f177d2a..c6f05e4 100644 --- a/test/support/pacer_doc_sample.ex +++ b/test/support/pacer_doc_sample.ex @@ -5,7 +5,7 @@ defmodule PacerDocSample do use Pacer.Workflow graph do - field(:a, doc: "this is a field that contains data about a thing") + field :a, doc: "this is a field that contains data about a thing" field(:undocumented_field, resolver: &__MODULE__.fetch/1, @@ -14,13 +14,12 @@ defmodule PacerDocSample do ) batch :api_requests do - field(:service_call, + field :service_call, default: nil, resolver: &__MODULE__.fetch/1, doc: "Fetches data from a service" - ) - field(:undocumented_service_call, default: nil, resolver: &__MODULE__.fetch/1) + field :undocumented_service_call, default: nil, resolver: &__MODULE__.fetch/1 end end From b5bf3267a3eda6f7677ad22010536df7eed66704 Mon Sep 17 00:00:00 2001 From: fishtreesugar Date: Wed, 5 Feb 2025 17:29:06 -0600 Subject: [PATCH 2/2] exclude virtual and redact fields in inspect result https://github.com/carsdotcom/pacer/issues/24 --- lib/workflow/dsl.ex | 13 +++++++++++-- lib/workflow/dsl/transformers/code_generation.ex | 5 +++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/workflow/dsl.ex b/lib/workflow/dsl.ex index 5b36efc..0e5b294 100644 --- a/lib/workflow/dsl.ex +++ b/lib/workflow/dsl.ex @@ -1,6 +1,6 @@ defmodule Pacer.Workflow.Dsl do defmodule Field do - defstruct [:name, :default, :dependencies, :resolver, :virtual?, :doc] + defstruct [:name, :default, :dependencies, :resolver, :virtual?, :doc, :redact?] end @field_schema [ @@ -117,6 +117,15 @@ defmodule Pacer.Workflow.Dsl do to avoid the extra memory overhead that would be associated with carrying these values downstream if, for example, the map returned from `Pacer.Workflow.execute/1` is stored in a long-lived process state; intermediate or transient values can cause unnecessary memory bloat if they are carried into process state where they are not neeeded. + + And virtual field will excluded in inspect result. + """ + ], + redact?: [ + default: false, + type: :boolean, + doc: """ + Similar to Ecto's `redact`, it will excluded in inspect result. """ ] ] @@ -130,7 +139,7 @@ defmodule Pacer.Workflow.Dsl do } defmodule BatchField do - defstruct [:name, :default, :dependencies, :resolver, :virtual?, :guard, :doc] + defstruct [:name, :default, :dependencies, :resolver, :virtual?, :guard, :doc, :redact?] end @batch_field_schema Keyword.merge( diff --git a/lib/workflow/dsl/transformers/code_generation.ex b/lib/workflow/dsl/transformers/code_generation.ex index 676adf6..156c42f 100644 --- a/lib/workflow/dsl/transformers/code_generation.ex +++ b/lib/workflow/dsl/transformers/code_generation.ex @@ -13,6 +13,7 @@ defmodule Pacer.Workflow.Dsl.Transformers.CodeGeneration do pacer_struct_fields: [], pacer_dependencies: [], pacer_virtual_fields: [], + pacer_redact_fields: [], pacer_resolvers: [] } @@ -32,6 +33,9 @@ defmodule Pacer.Workflow.Dsl.Transformers.CodeGeneration do |> update_in([:pacer_virtual_fields], fn v -> if field.virtual?, do: [field.name | v], else: v end) + |> update_in([:pacer_redact_fields], fn v -> + if field.redact?, do: [field.name | v], else: v + end) if not is_nil(batch) do info @@ -144,6 +148,7 @@ defmodule Pacer.Workflow.Dsl.Transformers.CodeGeneration do defstruct_ast = quote do + @derive {Inspect, except: unquote(info.pacer_virtual_fields ++ info.pacer_redact_fields)} defstruct unquote(Enum.reverse(info.pacer_struct_fields)) end