refactor: to allow connector/flow steps to interchange behaviours in the flow

classes/form/step_form.php

@@ -87,6 +87,10 @@ public function definition() {
         );
         $select->setMultiple(true);
 
+        // Is like a connector step? Does not operate within a flow group.
+        $mform->addElement('advcheckbox', 'connector', 'Behave like a connector step? (WIP)', false);
+        $mform->setType('connector', PARAM_BOOL);
+
         // List all the available fields available for configuration, in dot syntax.
         $variables = $this->get_available_references();
         $mform->addElement('html', $this->prepare_available_fields($variables));
@@ -361,6 +365,16 @@ protected function get_default_data() {
         if (!empty($type) && class_exists($type)) {
             $steptype = new $type();
             $data = $steptype->form_get_default_data($data);
+
+            // Automatically fill in the name with something hopefully sane for new steps.
+            if (empty($data->id)) {
+                $classname = $type;
+                $position = strrpos($classname, '\\');
+                $basename = substr($classname, $position + 1);
+                if ($position !== false) {
+                    $data->name = $basename;
+                }
+            }
         }
 
         return $data;

classes/local/execution/connector_engine_step.php

@@ -44,9 +44,9 @@ public function go(): int {
         switch ($this->proceed_status()) {
             case self::PROCEED_GO:
                 try {
-                    $result = $this->steptype->execute($this);
+                    $result = $this->steptype->execute(new \stdClass);
                     $this->steptype->prepare_outputs();
-                    if ($result === true) {
+                    if ($result !== false) {
                         $this->set_status(engine::STATUS_FINISHED);
                     } else {
                         $this->set_status(engine::STATUS_CANCELLED);

classes/local/execution/engine_step.php

@@ -293,4 +293,26 @@ protected function on_change_status() {
                 break;
         }
     }
+
+    /**
+     * Returns an array with all the variables available, with the context of the step
+     *
+     * @return  array
+     */
+    public function get_variables(): array {
+        // Config values are directly referenceable, step values go through
+        // step.fieldname, everything else is available through expressions,
+        // such as 'dataflow.id' and 'steps.mystep.name' for example.
+        $variables = $this->engine->get_variables();
+        $step = $variables['steps']->{$this->stepdef->alias};
+
+        // Pull out the config.
+        $config = $step->config;
+
+        return array_merge(
+            $variables,
+            ['step' => $step],
+            (array) $config,
+        );
+    }
 }

classes/local/execution/flow_engine_step.php

@@ -16,6 +16,8 @@
 
 namespace tool_dataflows\local\execution;
 
+use tool_dataflows\parser;
+
 /**
  * Manages the execution of a flow step.
  *
@@ -84,10 +86,24 @@ public function get_variables(): array {
         // such as 'dataflow.id' and 'steps.mystep.name' for example.
         $variables = $this->engine->get_variables();
         $step = $variables['steps']->{$this->stepdef->alias};
+
+        // Pull out the config.
+        $config = $step->config;
+
+        // Set the record as an available variable.
+        if ($this->iterator) {
+            $variables['record'] = (array) $this->iterator->current();
+
+            // Evaluate the config again with the record context, unless the
+            // step type doesn't want to (e.g. SQL reader does it own handling).
+            $parser = new parser;
+            $parser->evaluate_recursive($config, $variables);
+        }
+
         return array_merge(
             $variables,
             ['step' => $step],
-            (array) $step->config
+            (array) $config,
         );
     }
 }

classes/local/step/base_step.php

@@ -18,8 +18,10 @@
 
 use Symfony\Component\Yaml\Yaml;
 use tool_dataflows\helper;
+use tool_dataflows\local\execution\connector_engine_step;
 use tool_dataflows\local\execution\engine;
 use tool_dataflows\local\execution\engine_step;
+use tool_dataflows\local\execution\flow_engine_step;
 use tool_dataflows\parser;
 use tool_dataflows\step;
 
@@ -477,17 +479,26 @@ public function validate_for_run() {
      *
      * @return engine_step
      */
-    final public function get_engine_step(): engine_step {
+    final public function get_engine_step(): ?engine_step {
         return $this->enginestep;
     }
 
     /**
-     * Generate the engine step for the step type.
+     * Generates an engine step for this type.
+     *
+     * This should be sufficient for most cases. Override this function if needed.
      *
      * @param engine $engine
      * @return engine_step
      */
-    abstract protected function generate_engine_step(engine $engine): engine_step;
+    protected function generate_engine_step(engine $engine): engine_step {
+        // Determine if it should return a flow (iterator based) or connector (no iterator) engine step.
+        if ($this->is_flow()) {
+            return new flow_engine_step($engine, $this->stepdef, $this);
+        }
+
+        return new connector_engine_step($engine, $this->stepdef, $this);
+    }
 
     /**
      * Returns the group (string) this step type is categorised under.
@@ -607,4 +618,37 @@ public function get_output_label(int $position): string {
         $label = $labels[$position] ?? (string) $position;
         return $label;
     }
+
+    /**
+     * Get the step's (definition) current config.
+     *
+     * Helper method to reduce the complexity when authoring step types.
+     *
+     * @return  \stdClass configuration object
+     */
+    protected function get_config() {
+        return $this->enginestep->stepdef->config ?? $this->stepdef->config;
+    }
+
+    /**
+     * Returns whether the engine's run is dry
+     *
+     * Helper method to reduce the complexity when authoring step types.
+     *
+     * @return  bool mode is in dry run or not
+     */
+    protected function is_dry_run() {
+        return $this->enginestep->engine->isdryrun;
+    }
+
+    /**
+     * Emits a log message
+     *
+     * Helper method to reduce the complexity when authoring step types.
+     *
+     * @param string $message
+     */
+    protected function log($message) {
+        $this->enginestep->log($message);
+    }
 }

classes/local/step/connector_curl.php

@@ -44,7 +44,7 @@ class connector_curl extends connector_step {
      * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science)
      */
     public function has_side_effect(): bool {
-        if (isset($this->stepdef)) {
+        if (!empty($this->stepdef->id)) {
             $config = $this->stepdef->config;
 
             // Destination is outside of scratch directory.

classes/local/step/connector_s3.php

@@ -42,7 +42,7 @@ class connector_s3 extends connector_step {
      * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science)
      */
     public function has_side_effect(): bool {
-        if (isset($this->stepdef)) {
+        if (!empty($this->stepdef->id)) {
             $config = $this->stepdef->config;
             return !helper::path_is_relative($config->target);
         }

classes/local/step/connector_step.php

@@ -57,18 +57,6 @@ public function get_group(): string {
         return 'connectors';
     }
 
-    /**
-     * Generates an engine step for this type.
-     *
-     * This should be sufficient for most cases. Override this function if needed.
-     *
-     * @param engine $engine
-     * @return engine_step
-     */
-    protected function generate_engine_step(engine $engine): engine_step {
-        return new connector_engine_step($engine, $this->stepdef, $this);
-    }
-
     /**
      * Returns an array with styles used to draw the dot graph
      *