Merge pull request #857 from catalyst/improvements-from-35

improvements from 35

classes/form/step_form.php

@@ -107,11 +107,22 @@ public function definition() {
         $select->setMultiple(true);
 
         $dataflow = new dataflow($dataflowid);
-        $varroot = $dataflow->get_variables_root();
 
-        // List all the available fields available for configuration, in dot syntax.
-        $mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'),
-            $this->prepare_available_fields($varroot->get()));
+        try {
+            $varroot = $dataflow->get_variables_root();
+
+            // List all the available fields available for configuration, in dot syntax.
+            $mform->addElement(
+                'static',
+                'fields',
+                get_string('available_fields', 'tool_dataflows'),
+                $this->prepare_available_fields($varroot->get())
+            );
+        } catch (\Throwable $e) {
+            global $OUTPUT;
+            $errtext  = $OUTPUT->notification($e->getMessage());
+            $mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext);
+        }
 
         // Check and set custom form inputs if required. Defaulting to a
         // textarea config input for those not yet configured.

classes/local/execution/engine.php

@@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat
 
         // Find the flow blocks.
         $this->create_flow_caps();
+
+        // Make the runid available to the flow.
+        if (!$this->isdryrun) {
+            $variables = $this->get_variables();
+            $variables->set('run.name', $this->run->name);
+            $variables->set('run.id', $this->run->id);
+        }
     }
 
     /**
@@ -746,6 +753,12 @@ private function setup_logging() {
             $channel .= '/' . $this->run->name;
         }
 
+        // Set the starting time as 'now'.
+        $now = microtime(true);
+        [, $decimal] = explode('.', $now);
+        $decimal = substr($decimal, 0, 3); // Only use the first 3 digits after the decimal point.
+        $rundateformat = date("Ymd_His$decimal", $now);
+
         // Each channel represents a specific way of writing log information.
         $log = new Logger($channel);
 
@@ -785,26 +798,32 @@ private function setup_logging() {
         }
 
         // Dataflow run logger.
-        // e.g. '[dataroot]/tool_dataflows/3/21.log' as the path.
+        // Type: FILE_PER_RUN
+        // e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path.
         if (isset($loghandlers[log_handler::FILE_PER_RUN])) {
             $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
                 'tool_dataflows' . DIRECTORY_SEPARATOR .
-                $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name
-                . '.log';
+                $this->dataflow->id . DIRECTORY_SEPARATOR .
+                $rundateformat . '_' . $this->run->name . '.log';
 
             $streamhandler = new StreamHandler($dataflowrunlogpath, Logger::DEBUG);
             $streamhandler->setFormatter($lineformatter);
             $log->pushHandler($streamhandler);
         }
 
         // General dataflow logger (rotates daily to prevent big single log file).
-        // e.g. '[dataroot]/tool_dataflows/3-2006-01-02.log' as the path.
+        // Type: FILE_PER_DATAFLOW
+        // e.g. '[dataroot]/tool_dataflows/20060102-3.log' as the path.
         if (isset($loghandlers[log_handler::FILE_PER_DATAFLOW])) {
             $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
                 'tool_dataflows' . DIRECTORY_SEPARATOR .
                 $this->dataflow->id . '.log';
 
             $rotatingfilehandler = new RotatingFileHandler($dataflowlogpath, 0, Logger::DEBUG);
+            $dateformat = 'Ymd';
+            $filenameformat = '{date}_{filename}';
+            $rotatingfilehandler->setFilenameFormat($filenameformat, $dateformat);
+
             $rotatingfilehandler->setFormatter($lineformatter);
             $log->pushHandler($rotatingfilehandler);
         }

classes/local/step/connector_find.php

@@ -0,0 +1,92 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Find (a record from a collection)
+ *
+ * @package    tool_dataflows
+ * @author     Kevin Pham <[email protected]>
+ * @copyright  Catalyst IT, 2022
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_find extends connector_step {
+
+    /**
+     * Returns whether the step has a side effect.
+     *
+     * @return     bool whether or not this step has a side effect
+     * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science)
+     */
+    public function has_side_effect(): bool {
+        return false;
+    }
+
+    /**
+     * Return the definition of the fields available in this form.
+     *
+     * @return array
+     */
+    public static function form_define_fields(): array {
+        return [
+            'collection' => ['type' => PARAM_TEXT, 'required' => true],
+            'condition' => ['type' => PARAM_TEXT, 'required' => true],
+        ];
+    }
+
+    /**
+     * Allows each step type to determine a list of optional/required form
+     * inputs for their configuration
+     *
+     * It's recommended you prefix the additional config related fields to avoid
+     * conflicts with any existing fields.
+     *
+     * @param \MoodleQuickForm $mform
+     */
+    public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
+        $mform->addElement('text', 'config_collection', get_string('find:collection', 'tool_dataflows'));
+        $mform->addElement('static', 'config_collection_help', '', get_string('find:collection_help', 'tool_dataflows'));
+        $mform->addElement('text', 'config_condition', get_string('find:condition', 'tool_dataflows'));
+        $mform->addElement('static', 'config_condition_help', '', get_string('find:condition_help', 'tool_dataflows'));
+    }
+
+    /**
+     * Find based on the condition, and set the matching record given a record.
+     *
+     * If the message is empty, it will not log anything, which is useful for conditional logging.
+     *
+     * @param mixed|null $input
+     * @return mixed
+     */
+    public function execute($input = null) {
+        $variables = $this->get_variables();
+        $config = $variables->get('config');
+        $collection = $config->collection;
+
+        foreach ($collection as $item) {
+            $variables->set('record', $item);
+            $condition = $variables->evaluate('${{'.$config->condition.'}}');
+            if ($condition) {
+                $input = $item;
+                break;
+            }
+        }
+        $variables->set('match', $input);
+
+        return $input;
+    }
+}

classes/local/step/connector_set_multiple_variables.php

@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Set multiple variables connector step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_set_multiple_variables extends connector_step {
+    use set_multiple_variables_trait;
+
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max). */
+    protected $outputconnectors = [0, 1];
+}

classes/local/step/connector_sql.php

@@ -0,0 +1,29 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * SQL connector step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_sql extends connector_step {
+    use sql_trait;
+}

classes/local/step/connector_update_user.php

@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Update user connector step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_update_user extends connector_step {
+    use update_user_trait;
+
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max). */
+    protected $outputconnectors = [0, 1];
+}

classes/local/step/copy_file_trait.php

@@ -72,7 +72,9 @@ public function execute($input = null) {
         $todirectory = dirname($to);
         if (!file_exists($todirectory)) {
             $this->log("Creating a directory at {$todirectory}");
-            mkdir($todirectory, $CFG->directorypermissions, true);
+            if (!mkdir($todirectory, $CFG->directorypermissions, true)) {
+                throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory);
+            }
         }
 
         // Attempt to copy the file to the destination.
@@ -112,10 +114,12 @@ public function execute($input = null) {
     private function copy(string $from, string $to) {
         $this->log("Copying $from to $to");
         if (!copy($from, $to)) {
-            throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [
-                'from' => $from,
-                'to' => $to,
-            ]);
+            throw new \moodle_exception(
+                'flow_copy_file:copy_failed',
+                'tool_dataflows',
+                '',
+                (object) ['from' => $from, 'to' => $to]
+            );
         }
     }
 

classes/local/step/flow_email.php

@@ -28,6 +28,12 @@
  */
 class flow_email extends flow_step {
 
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max) */
+    protected $outputconnectors = [0, 1];
+
     /** @var bool whether or not this step type (potentially) contains a side effect or not */
     protected $hassideeffect = true;
 

classes/local/step/flow_set_multiple_variables.php

@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Set multiple variables flow step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class flow_set_multiple_variables extends flow_step {
+    use set_multiple_variables_trait;
+
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max). */
+    protected $outputconnectors = [0, 1];
+}