diff --git a/classes/form/step_form.php b/classes/form/step_form.php index 5eec7d04..7e3af96e 100644 --- a/classes/form/step_form.php +++ b/classes/form/step_form.php @@ -107,11 +107,22 @@ public function definition() { $select->setMultiple(true); $dataflow = new dataflow($dataflowid); - $varroot = $dataflow->get_variables_root(); - // List all the available fields available for configuration, in dot syntax. - $mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'), - $this->prepare_available_fields($varroot->get())); + try { + $varroot = $dataflow->get_variables_root(); + + // List all the available fields available for configuration, in dot syntax. + $mform->addElement( + 'static', + 'fields', + get_string('available_fields', 'tool_dataflows'), + $this->prepare_available_fields($varroot->get()) + ); + } catch (\Throwable $e) { + global $OUTPUT; + $errtext = $OUTPUT->notification($e->getMessage()); + $mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext); + } // Check and set custom form inputs if required. Defaulting to a // textarea config input for those not yet configured. diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php index ae845cbd..1494178e 100644 --- a/classes/local/execution/engine.php +++ b/classes/local/execution/engine.php @@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat // Find the flow blocks. $this->create_flow_caps(); + + // Make the runid available to the flow. + if (!$this->isdryrun) { + $variables = $this->get_variables(); + $variables->set('run.name', $this->run->name); + $variables->set('run.id', $this->run->id); + } } /** @@ -746,6 +753,12 @@ private function setup_logging() { $channel .= '/' . $this->run->name; } + // Set the starting time as 'now'. + $now = microtime(true); + [, $decimal] = explode('.', $now); + $decimal = substr($decimal, 0, 3); // Only use the first 3 digits after the decimal point. + $rundateformat = date("Ymd_His$decimal", $now); + // Each channel represents a specific way of writing log information. $log = new Logger($channel); @@ -785,12 +798,13 @@ private function setup_logging() { } // Dataflow run logger. - // e.g. '[dataroot]/tool_dataflows/3/21.log' as the path. + // Type: FILE_PER_RUN + // e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path. if (isset($loghandlers[log_handler::FILE_PER_RUN])) { $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . - $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name - . '.log'; + $this->dataflow->id . DIRECTORY_SEPARATOR . + $rundateformat . '_' . $this->run->name . '.log'; $streamhandler = new StreamHandler($dataflowrunlogpath, Logger::DEBUG); $streamhandler->setFormatter($lineformatter); @@ -798,13 +812,18 @@ private function setup_logging() { } // General dataflow logger (rotates daily to prevent big single log file). - // e.g. '[dataroot]/tool_dataflows/3-2006-01-02.log' as the path. + // Type: FILE_PER_DATAFLOW + // e.g. '[dataroot]/tool_dataflows/20060102-3.log' as the path. if (isset($loghandlers[log_handler::FILE_PER_DATAFLOW])) { $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . $this->dataflow->id . '.log'; $rotatingfilehandler = new RotatingFileHandler($dataflowlogpath, 0, Logger::DEBUG); + $dateformat = 'Ymd'; + $filenameformat = '{date}_{filename}'; + $rotatingfilehandler->setFilenameFormat($filenameformat, $dateformat); + $rotatingfilehandler->setFormatter($lineformatter); $log->pushHandler($rotatingfilehandler); } diff --git a/classes/local/step/connector_find.php b/classes/local/step/connector_find.php new file mode 100644 index 00000000..1a51c6cd --- /dev/null +++ b/classes/local/step/connector_find.php @@ -0,0 +1,92 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Find (a record from a collection) + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2022 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_find extends connector_step { + + /** + * Returns whether the step has a side effect. + * + * @return bool whether or not this step has a side effect + * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) + */ + public function has_side_effect(): bool { + return false; + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'collection' => ['type' => PARAM_TEXT, 'required' => true], + 'condition' => ['type' => PARAM_TEXT, 'required' => true], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_collection', get_string('find:collection', 'tool_dataflows')); + $mform->addElement('static', 'config_collection_help', '', get_string('find:collection_help', 'tool_dataflows')); + $mform->addElement('text', 'config_condition', get_string('find:condition', 'tool_dataflows')); + $mform->addElement('static', 'config_condition_help', '', get_string('find:condition_help', 'tool_dataflows')); + } + + /** + * Find based on the condition, and set the matching record given a record. + * + * If the message is empty, it will not log anything, which is useful for conditional logging. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + $variables = $this->get_variables(); + $config = $variables->get('config'); + $collection = $config->collection; + + foreach ($collection as $item) { + $variables->set('record', $item); + $condition = $variables->evaluate('${{'.$config->condition.'}}'); + if ($condition) { + $input = $item; + break; + } + } + $variables->set('match', $input); + + return $input; + } +} diff --git a/classes/local/step/connector_set_multiple_variables.php b/classes/local/step/connector_set_multiple_variables.php new file mode 100644 index 00000000..b609490b --- /dev/null +++ b/classes/local/step/connector_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_set_multiple_variables extends connector_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/connector_sql.php b/classes/local/step/connector_sql.php new file mode 100644 index 00000000..0cb01e5d --- /dev/null +++ b/classes/local/step/connector_sql.php @@ -0,0 +1,29 @@ +. + +namespace tool_dataflows\local\step; + +/** + * SQL connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_sql extends connector_step { + use sql_trait; +} diff --git a/classes/local/step/connector_update_user.php b/classes/local/step/connector_update_user.php new file mode 100644 index 00000000..01e569df --- /dev/null +++ b/classes/local/step/connector_update_user.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_update_user extends connector_step { + use update_user_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/copy_file_trait.php b/classes/local/step/copy_file_trait.php index 30a081c5..05cc866b 100644 --- a/classes/local/step/copy_file_trait.php +++ b/classes/local/step/copy_file_trait.php @@ -72,7 +72,9 @@ public function execute($input = null) { $todirectory = dirname($to); if (!file_exists($todirectory)) { $this->log("Creating a directory at {$todirectory}"); - mkdir($todirectory, $CFG->directorypermissions, true); + if (!mkdir($todirectory, $CFG->directorypermissions, true)) { + throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory); + } } // Attempt to copy the file to the destination. @@ -112,10 +114,12 @@ public function execute($input = null) { private function copy(string $from, string $to) { $this->log("Copying $from to $to"); if (!copy($from, $to)) { - throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [ - 'from' => $from, - 'to' => $to, - ]); + throw new \moodle_exception( + 'flow_copy_file:copy_failed', + 'tool_dataflows', + '', + (object) ['from' => $from, 'to' => $to] + ); } } diff --git a/classes/local/step/flow_email.php b/classes/local/step/flow_email.php index 7090dfdd..d35d1510 100644 --- a/classes/local/step/flow_email.php +++ b/classes/local/step/flow_email.php @@ -28,6 +28,12 @@ */ class flow_email extends flow_step { + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max) */ + protected $outputconnectors = [0, 1]; + /** @var bool whether or not this step type (potentially) contains a side effect or not */ protected $hassideeffect = true; diff --git a/classes/local/step/flow_set_multiple_variables.php b/classes/local/step/flow_set_multiple_variables.php new file mode 100644 index 00000000..1115240e --- /dev/null +++ b/classes/local/step/flow_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables flow step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_set_multiple_variables extends flow_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/flow_sql.php b/classes/local/step/flow_sql.php index 0bfff0ca..1e91ec76 100644 --- a/classes/local/step/flow_sql.php +++ b/classes/local/step/flow_sql.php @@ -27,131 +27,4 @@ class flow_sql extends flow_step { use sql_trait; - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'sql' => ['type' => PARAM_TEXT, 'required' => true], - ]; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - // SQL example and inputs. - $sqlexample = " - SELECT id, username - FROM {user} - WHERE id > \${{steps.xyz.var.number}} - ORDER BY id ASC - LIMIT 10"; - $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); - $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), - ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); - $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); - } - - /** - * Allow steps to setup the form depending on current values. - * - * This method is called after definition(), data submission and set_data(). - * All form setup that is dependent on form values should go in here. - * - * @param \MoodleQuickForm $mform - * @param \stdClass $data - */ - public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { - // Validate the data. - $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); - - // Get the element. - $element = $mform->getElement('config_sql'); - - // Update the element height based on min/max settings, but preserve - // other existing rules. - $attributes = $element->getAttributes(); - - // Set the rows at a minimum to the predefined amount in - // form_add_custom_inputs, and expand as content grows up to a maximum. - $attributes['rows'] = min( - $attributes['max_rows'], - max($attributes['rows'], $sqllinecount) - ); - $element->setAttributes($attributes); - } - - /** - * Execute configured query - * - * @param mixed $input - * @return mixed - * @throws \dml_read_exception when the SQL is not valid. - */ - public function execute($input = null) { - global $DB; - - // Construct the query. - $variables = $this->get_variables(); - $config = $variables->get_raw('config'); - [$sql, $params] = $this->evaluate_expressions($config->sql); - - // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. - // This is not used for security, just to route the request via the correct pathway for readonly databases. - $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; - $matches = []; - preg_match($pattern, $sql, $matches); - - // Matches[0] contains the match. Fallthrough to default on no match. - $token = $matches[0] ?? ''; - $emptydefault = new \stdClass(); - - switch(strtoupper($token)) { - case 'SELECT': - // Execute the query using get_records instead of get_record. - // This is so we can expose the number of records returned which - // can then be used by the dataflow in for e.g. a switch statement. - $records = $DB->get_records_sql($sql, $params); - - $variables->set('count', count($records)); - $invalidnum = ($records === false || count($records) !== 1); - $data = $invalidnum ? $emptydefault : array_pop($records); - $variables->set('data', $data); - break; - default: - // Default to execute. - $success = $DB->execute($sql, $params); - - // We can't really do anything with the response except check for success. - $variables->set('count', (int) $success); - $variables->set('data', $emptydefault); - break; - } - - return $input; - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (empty($config->sql)) { - $errors['config_sql'] = get_string('config_field_missing', 'tool_dataflows', 'sql', true); - } - - return empty($errors) ? true : $errors; - } } diff --git a/classes/local/step/flow_transformer_regex.php b/classes/local/step/flow_transformer_regex.php index d4261157..1f50c2b0 100644 --- a/classes/local/step/flow_transformer_regex.php +++ b/classes/local/step/flow_transformer_regex.php @@ -62,6 +62,7 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) { get_string('flow_transformer_regex:pattern', 'tool_dataflows'), [ 'placeholder' => "/[abc]/", + 'size' => '60', ] ); $mform->addElement( diff --git a/classes/local/step/flow_update_user.php b/classes/local/step/flow_update_user.php new file mode 100644 index 00000000..023b9195 --- /dev/null +++ b/classes/local/step/flow_update_user.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user flow step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_update_user extends flow_step { + use update_user_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/reader_csv.php b/classes/local/step/reader_csv.php index c5f8e7aa..1de5e7f0 100644 --- a/classes/local/step/reader_csv.php +++ b/classes/local/step/reader_csv.php @@ -43,6 +43,7 @@ public static function form_define_fields(): array { 'path' => ['type' => PARAM_TEXT, 'required' => true], 'headers' => ['type' => PARAM_TEXT], 'overwriteheaders' => ['type' => PARAM_BOOL], + 'continueonerror' => ['type' => PARAM_BOOL], 'delimiter' => ['type' => PARAM_TEXT], ]; } @@ -61,9 +62,11 @@ public function get_iterator(): iterator { */ public function csv_contents_generator() { $maxlinelength = 1000; - $config = $this->get_variables()->get('config'); + $variables = $this->get_variables(); + $config = $variables->get('config'); $strheaders = $config->headers; $overwriteheaders = !empty($config->overwriteheaders); + $continueonerror = !empty($config->continueonerror); $delimiter = $config->delimiter ?: self::DEFAULT_DELIMETER; $path = $this->enginestep->engine->resolve_path($config->path); @@ -92,12 +95,35 @@ public function csv_contents_generator() { // Convert header string to an actual headers array. $headers = str_getcsv($strheaders, $delimiter); $numheaders = count($headers); + $rownumber = 1; // First row is always headers. + $errors = ['header_field_count_mismatch' => 0]; while (($data = fgetcsv($handle, $maxlinelength, $delimiter)) !== false) { + $rownumber++; $numfields = count($data); if ($numfields !== $numheaders) { + // Continue on (parse) error. + if ($continueonerror) { + $errors['header_field_count_mismatch'] += 1; + $this->log->error( + get_string('reader_csv:header_field_count_mismatch', 'tool_dataflows', (object) [ + 'numfields' => $numfields, + 'numheaders' => $numheaders, + 'rownumber' => $rownumber, + ]), + [ + 'fields' => $data, + 'headers' => $headers, + ] + ); + + continue; + } + + // Throw exception on error. throw new \moodle_exception('reader_csv:header_field_count_mismatch', 'tool_dataflows', '', (object) [ 'numfields' => $numfields, 'numheaders' => $numheaders, + 'rownumber' => $rownumber, ], json_encode([ 'fields' => $data, 'headers' => $headers, @@ -109,6 +135,8 @@ public function csv_contents_generator() { } finally { fclose($handle); } + + $variables->set('errors', (object) $errors); } /** @@ -150,6 +178,10 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) { $mform->hideIf('config_overwriteheaders', 'config_headers', 'eq', ''); $mform->disabledIf('config_overwriteheaders', 'config_headers', 'eq', ''); + // Used when we want to replace the headers (or overwrite them) using the ones we supplied instead. Defaults to off. + $mform->addElement('checkbox', 'config_continueonerror', get_string('reader_csv:continueonerror', 'tool_dataflows'), + get_string('reader_csv:continueonerror_help', 'tool_dataflows')); + // Delimiter. $mform->addElement( 'text', diff --git a/classes/local/step/set_multiple_variables_trait.php b/classes/local/step/set_multiple_variables_trait.php new file mode 100644 index 00000000..6d2ea232 --- /dev/null +++ b/classes/local/step/set_multiple_variables_trait.php @@ -0,0 +1,101 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\local\variables\var_root; +use tool_dataflows\local\variables\var_object_visible; + +/** + * Set multiple variables trait + * + * Similar to the single approach, except it allows multiple to be set in a + * single step. This is great for initialising counters, and initial variables + * that need to be reset every run, but might change during. + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait set_multiple_variables_trait { + use set_variable_trait; + + /** + * Executes the step, fetching the config and actioning the step. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + + $stepvars = $this->get_variables(); + $config = $stepvars->get('config'); + $rootvars = $this->get_variables_root(); + $this->run($rootvars, $config->field, $config->values); + + return $input; + } + + /** + * Main handler of this step, split out to make it easier to test. + * + * @param var_root $varobject + * @param string $field + * @param mixed $values + */ + public function run(var_root $varobject, string $field, $values) { + // Do nothing if the value has not changed. + $currentvalue = $varobject->get($field); + if ($currentvalue === $values) { + return; + } + + // Set the value in the variable tree. + $varobject->set($field, $values); + $this->log->info("Set '{field}' as '{values}'", ['field' => $field, 'values' => json_encode($values)]); + + // We do not persist the value if it is a dry run. + if ($this->is_dry_run()) { + return; + } + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'field' => ['type' => PARAM_TEXT, 'required' => true], + 'values' => ['type' => PARAM_TEXT, 'required' => true, 'yaml' => true], + ]; + } + + /** + * Custom form inputs + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_field', get_string('set_multiple_variables:field', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:field_help', 'tool_dataflows')); + + $mform->addElement('textarea', 'config_values', get_string('set_multiple_variables:values', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:values_help', 'tool_dataflows')); + } +} diff --git a/classes/local/step/sftp_trait.php b/classes/local/step/sftp_trait.php index 1304dea4..0c61c53b 100644 --- a/classes/local/step/sftp_trait.php +++ b/classes/local/step/sftp_trait.php @@ -35,10 +35,13 @@ trait sftp_trait { /** Shorthand sftp scheme for use in config. */ - static protected $sftpprefix = 'sftp'; + protected static $sftpprefix = 'sftp'; /** Default port to connect to. */ - static protected $defaultport = 22; + protected static $defaultport = 22; + + /** Array of SFTP objects to use for performance reasons. */ + protected $sftp = []; /** * Returns whether or not the step configured, has a side effect. @@ -93,17 +96,14 @@ public function form_add_core_inputs(\MoodleQuickForm &$mform) { $mform->addElement('text', 'config_port', get_string('connector_sftp:port', 'tool_dataflows')); $mform->setDefault('config_port', self::$defaultport); $mform->addElement('text', 'config_hostpubkey', get_string('connector_sftp:hostpubkey', 'tool_dataflows')); - $mform->addElement('static', 'config_hostpubkey_desc', '', - get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_hostpubkey_desc', '', get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); $mform->addElement('text', 'config_username', get_string('username')); $mform->addElement('passwordunmask', 'config_password', get_string('password')); - $mform->addElement('static', 'config_password_desc', '', - get_string('connector_sftp:password_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_password_desc', '', get_string('connector_sftp:password_desc', 'tool_dataflows')); $mform->addElement('text', 'config_privkeyfile', get_string('connector_sftp:privkeyfile', 'tool_dataflows')); - $mform->addElement('static', 'config_keyfile_desc', '', - get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_keyfile_desc', '', get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); } /** @@ -118,13 +118,19 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform, $behaviour = 'c if ($behaviour === 'copy') { $mform->addElement('text', 'config_source', get_string('connector_sftp:source', 'tool_dataflows')); $mform->addElement('static', 'config_source_desc', '', get_string('connector_sftp:source_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); $mform->addElement('text', 'config_target', get_string('connector_sftp:target', 'tool_dataflows')); $mform->addElement('static', 'config_target_desc', '', get_string('connector_sftp:target_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); } } @@ -247,11 +253,8 @@ public function execute($input = null) { $stepvars = $this->get_variables(); $config = $stepvars->get('config'); - $this->log->debug("Connecting to {$config->host}:{$config->port}"); - // At this point we need to disconnect once we are finished. try { - // Skip if it is a dry run. if ($this->is_dry_run() && $this->has_side_effect()) { return $input; @@ -278,7 +281,7 @@ public function execute($input = null) { // Upload to remote. $this->upload($sftp, $sourcepath, $targetpath); - } finally { + } catch (\Throwable $e) { if (isset($sftp)) { $sftp->disconnect(); } @@ -287,6 +290,28 @@ public function execute($input = null) { return $input; } + /** + * Hook function that gets called when an engine step has been aborted. + */ + public function on_abort() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + + /** + * Hook function that gets called when an engine step has been finalised. + */ + public function on_finalise() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + /** * Checks and loads the appropriate key, based on config * @@ -408,6 +433,14 @@ public function resolve_path(string $pathname): string { * @return SFTP */ private function init_sftp($config): SFTP { + // Use existing cached SFTP object if available. + $cachekey = implode('|', [$config->host, $config->port, $config->username]); + if (isset($this->sftp[$cachekey])) { + return $this->sftp[$cachekey]; + } + + // Create and connect to SFTP. + $this->log->debug("Connecting to {$config->host}:{$config->port}"); $sftp = new SFTP($config->host, $config->port); $this->check_public_host_key($sftp, $config->hostpubkey); @@ -417,6 +450,10 @@ private function init_sftp($config): SFTP { } $sftp->enableDatePreservation(); + + // Cache sftp since it takes a while to attempt initial connection. + $this->sftp[$cachekey] = $sftp; + return $sftp; } } diff --git a/classes/local/step/sql_trait.php b/classes/local/step/sql_trait.php index 5b1157cc..e3c13762 100644 --- a/classes/local/step/sql_trait.php +++ b/classes/local/step/sql_trait.php @@ -75,8 +75,8 @@ private function evaluate_expressions(string $sql) { [$hasexpression] = $parser->has_expression($el); $max--; } - if (!in_array(gettype($el), ['string', 'int'])) { - throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows'); + if (!in_array(gettype($el), ['string', 'int', 'integer'])) { + throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows', '', gettype($el)); } return $el; }, $expressions); @@ -167,4 +167,117 @@ public function validate_config($config) { } return empty($errors) ? true : $errors; } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'sql' => ['type' => PARAM_TEXT, 'required' => true], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + // SQL example and inputs. + $sqlexample = " + SELECT id, username + FROM {user} + WHERE id > \${{steps.xyz.var.number}} + ORDER BY id ASC + LIMIT 10"; + $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); + $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), + ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); + $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); + } + + /** + * Allow steps to setup the form depending on current values. + * + * This method is called after definition(), data submission and set_data(). + * All form setup that is dependent on form values should go in here. + * + * @param \MoodleQuickForm $mform + * @param \stdClass $data + */ + public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { + // Validate the data. + $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); + + // Get the element. + $element = $mform->getElement('config_sql'); + + // Update the element height based on min/max settings, but preserve + // other existing rules. + $attributes = $element->getAttributes(); + + // Set the rows at a minimum to the predefined amount in + // form_add_custom_inputs, and expand as content grows up to a maximum. + $attributes['rows'] = min( + $attributes['max_rows'], + max($attributes['rows'], $sqllinecount) + ); + $element->setAttributes($attributes); + } + + /** + * Execute configured query + * + * @param mixed $input + * @return mixed + * @throws \dml_read_exception when the SQL is not valid. + */ + public function execute($input = null) { + global $DB; + + // Construct the query. + $variables = $this->get_variables(); + $config = $variables->get_raw('config'); + [$sql, $params] = $this->evaluate_expressions($config->sql); + + // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. + // This is not used for security, just to route the request via the correct pathway for readonly databases. + $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; + $matches = []; + preg_match($pattern, $sql, $matches); + + // Matches[0] contains the match. Fallthrough to default on no match. + $token = $matches[0] ?? ''; + $emptydefault = new \stdClass(); + + switch(strtoupper($token)) { + case 'SELECT': + // Execute the query using get_records instead of get_record. + // This is so we can expose the number of records returned which + // can then be used by the dataflow in for e.g. a switch statement. + $records = $DB->get_records_sql($sql, $params); + + $variables->set('count', count($records)); + $invalidnum = ($records === false || count($records) !== 1); + $data = $invalidnum ? $emptydefault : array_pop($records); + $variables->set('data', $data); + break; + default: + // Default to execute. + $success = $DB->execute($sql, $params); + + // We can't really do anything with the response except check for success. + $variables->set('count', (int) $success); + $variables->set('data', $emptydefault); + break; + } + + return $input; + } } diff --git a/classes/local/step/update_user_trait.php b/classes/local/step/update_user_trait.php new file mode 100644 index 00000000..e5204263 --- /dev/null +++ b/classes/local/step/update_user_trait.php @@ -0,0 +1,93 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user using core api + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait update_user_trait { + + /** + * Returns whether or not the step configured, has a side effect. + * + * @return bool whether or not this step has a side effect + * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) + */ + public function has_side_effect(): bool { + return true; + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'userid' => ['type' => PARAM_TEXT, 'required' => true], + 'fields' => ['type' => PARAM_TEXT, 'required' => true, 'yaml' => true], + ]; + } + + /** + * Custom elements for editing the connector. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_userid', get_string('update_user:userid', 'tool_dataflows')); + $mform->addElement('static', 'config_userid_help', '', get_string('update_user:userid_help', 'tool_dataflows')); + + $mform->addElement( + 'textarea', + 'config_fields', + get_string('update_user:fields', 'tool_dataflows'), + ['cols' => 60, 'rows' => 5] + ); + $mform->addElement('static', 'config_fields_help', '', get_string('update_user:fields_help', 'tool_dataflows')); + } + + /** + * Main execution method which updates the user's details. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + $variables = $this->get_variables(); + $config = $variables->get('config'); + $userobject = (object) array_merge( + ['id' => $config->userid], + (array) $config->fields + ); + + // Some fields will not be updatable (intentionally). + unset($userobject->password); + unset($userobject->auth); + + // Update user fields using core api. + user_update_user($userobject, false, false); + profile_save_data($userobject); + + return $input; + } +} diff --git a/db/upgrade.php b/db/upgrade.php index 90e076c0..29a56016 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -287,5 +287,49 @@ function xmldb_tool_dataflows_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2023072100, 'tool', 'dataflows'); } + // Move log files that exist across to new format. Breaking change if any + // dataflows implement logic based on these files based on filename format. + if ($oldversion < 2023110901) { + $path = '*.log'; + $pattern = '/(\d+)-(\d{4})-(\d{2})-(\d{2})/m'; + $replace = '$2$3$4_$1'; + xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace); + + $path = '*/*.log'; + $pattern = '/(\d+)\/(\d+)(_)*.*\.log/m'; + $replace = '$1/{modifiedtime}_$2.log'; + xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace, true); + + // Dataflows savepoint reached. + upgrade_plugin_savepoint(true, 2023110901, 'tool', 'dataflows'); + } + return true; } + +/** + * Log file helper function + * + * @param string $path + * @param string $pattern + * @param string $replace + * @param bool $modifiedtimeprefix whether or not to add a datetime prefix to the new log file + * @return bool result + */ +function xmldb_tool_dataflows_logfile_rename_helper(string $path, string $pattern, string $replace, $modifiedtimeprefix = false) { + global $CFG; + + $plugindatadir = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows'; + $files = glob($plugindatadir . DIRECTORY_SEPARATOR . $path); + foreach ($files as $file) { + $strreplace = $replace; + if ($modifiedtimeprefix) { + $newprefix = date('Ymd_His000', filemtime($file)); + $strreplace = str_replace('{modifiedtime}', $newprefix, $replace); + } + $newlocation = preg_replace($pattern, $strreplace, $file); + if ($newlocation) { + rename($file, $newlocation); + } + } +} diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 97ec6645..ba22bc0f 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -47,8 +47,8 @@ $string['gpg_key_dir_desc'] = 'Path to keyring directory'; $string['log_handlers'] = 'Log handlers'; $string['log_handlers_desc'] = 'Additional log handlers to output dataflow logs to more destinations. The handler for mtrace is always active and cannot be disabled. Applying the settings at the dataflow level will override settings applied at the site admin level.'; -$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/{id}-Y-m-d.log'; -$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/{id}.log'; +$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/Ymd_{dataflowid}.log'; +$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/Ymd_his_{runid}.log'; $string['log_handler_browser_console'] = 'Browser Console'; $string['permitted_dirs'] = 'Permitted directories'; $string['permitted_dirs_desc'] = "List directories here to allow them to be read from/written to by dataflow steps. @@ -145,6 +145,7 @@ $string['step_name_connector_remove_file'] = 'Remove file'; $string['step_name_connector_s3'] = 'S3 file copy'; $string['step_name_connector_set_variable'] = 'Set variable'; +$string['step_name_connector_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_connector_sftp'] = 'SFTP file copy'; $string['step_name_connector_sns_notify'] = 'AWS-SNS Notification'; $string['step_name_connector_wait'] = 'Wait'; @@ -159,6 +160,7 @@ $string['step_name_flow_hash_file'] = 'Hash file'; $string['step_name_flow_remove_file'] = 'Remove file'; $string['step_name_flow_set_variable'] = 'Set variable'; +$string['step_name_flow_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_flow_logic_join'] = 'Join'; $string['step_name_flow_logic_switch'] = 'Switch'; $string['step_name_flow_noop'] = 'No-op'; @@ -179,6 +181,7 @@ $string['step_name_writer_stream'] = 'Stream writer'; $string['step_name_trigger_event'] = 'Moodle event'; $string['step_name_flow_sql'] = 'SQL'; +$string['step_name_connector_sql'] = 'SQL'; // Step (type) groups. $string['stepgrouptriggers'] = 'Triggers'; @@ -350,7 +353,7 @@ $string['writer_csv:fail_to_encode'] = 'Failed to encode CSV.'; // SQL trait. -$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int).'; +$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int), found {$a}'; $string['sql_trait:variable_not_valid_in_position_replacement_text'] = "Invalid expression \${{ {\$a->expression} }} as `{\$a->expressionpath}` could not be resolved at line {\$a->line} character {\$a->column} in:\n{\$a->sql}"; // phpcs:disable moodle.Strings.ForbiddenStrings.Found // Reader SQL. @@ -370,7 +373,9 @@ $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.'; $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers'; $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.'; -$string['reader_csv:header_field_count_mismatch'] = 'Number of headers ({$a->numheaders}) should match number of fields ({$a->numfields})'; +$string['reader_csv:continueonerror'] = 'Continue on parsing errors'; +$string['reader_csv:continueonerror_help'] = 'If checked, the step will continue reading the next row of data if there are parsing errors on the current.'; +$string['reader_csv:header_field_count_mismatch'] = 'Row #{$a->rownumber}: Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; // Reader JSON. $string['reader_json:arrayexpression_help'] = 'Nested array to extract from JSON. For example, {$a->expression} will return the users array from the following JSON (If empty it is assumed the starting point of the JSON file is an array):{$a->jsonexample}'; @@ -546,6 +551,7 @@ $string['flow_copy_file:from'] = 'From'; $string['flow_copy_file:to'] = 'To'; $string['flow_copy_file:copy_failed'] = 'Failed to copy {$a->from} to {$a->to}'; +$string['flow_copy_file:mkdir_failed'] = 'Failed to create directory at {$a}. Please check permissions and try again.'; // Directory file count. $string['connector_directory_file_count:path'] = 'Path to directory'; @@ -588,6 +594,12 @@ $string['set_variable:value'] = 'Value'; $string['set_variable:value_help'] = 'The value could be a number, text, or an expression. For example: ${{ record.id }}.'; +// Set multiple variables step. +$string['set_multiple_variables:field'] = 'Field'; +$string['set_multiple_variables:field_help'] = 'Defines the path to the field you would like to set the value(s). For example: dataflow.vars.counter.'; +$string['set_multiple_variables:values'] = 'Values'; +$string['set_multiple_variables:values_help'] = 'A list of fields/keys and values, in YAML format.'; + // Event trigger. $string['trigger_event:policy:immediate'] = 'Run immediately'; $string['trigger_event:policy:adhoc'] = 'Run ASAP in individual tasks in parallel'; @@ -622,3 +634,18 @@ $string['log:level'] = 'Log Level'; $string['log:level_help'] = 'Choose between 8 logging levels. Each level conveys a different meaning, and it is best practice to choose the most appropriate one for the log message.'; $string['log:message'] = 'Message'; + +// Find step. +$string['step_name_connector_find'] = 'Find'; +$string['find:collection'] = 'Collection'; +$string['find:collection_help'] = 'A reference to a collection of items.'; +$string['find:condition'] = 'Condition'; +$string['find:condition_help'] = 'The expression checked against each item in the referenced collection. For each item in the collection, it can be referenced as "item.field"'; + +// Update user step. +$string['step_name_connector_update_user'] = 'Update user'; +$string['step_name_flow_update_user'] = 'Update user'; +$string['update_user:userid'] = 'User ID'; +$string['update_user:userid_help'] = 'The internal ID for this user.'; +$string['update_user:fields'] = 'Fields'; +$string['update_user:fields_help'] = 'Fields set here correspond to the fields available in the underlying user table. Custom user profile fields, must be prefixed with "profile_field_" followed by the shortname. Not all fields can be updated, such as password and auth'; diff --git a/lib.php b/lib.php index 0f9404e2..f5bf130a 100644 --- a/lib.php +++ b/lib.php @@ -60,6 +60,7 @@ function tool_dataflows_step_types() { new step\connector_email, new step\connector_file_exists, new step\connector_file_put_content, + new step\connector_find, new step\connector_gpg, new step\connector_hash_file, new step\connector_log, @@ -68,6 +69,8 @@ function tool_dataflows_step_types() { new step\connector_s3, new step\connector_set_variable, new step\connector_sftp, + new step\connector_sql, + new step\connector_update_user, new step\connector_sftp_directory_file_list, new step\connector_sns_notify, new step\connector_wait, @@ -92,6 +95,7 @@ function tool_dataflows_step_types() { new step\flow_transformer_alter, new step\flow_transformer_filter, new step\flow_transformer_regex, + new step\flow_update_user, new step\flow_web_service, new step\reader_csv, new step\reader_directory_file_list, @@ -103,6 +107,8 @@ function tool_dataflows_step_types() { new step\trigger_webservice, new step\writer_debugging, new step\writer_stream, + new step\connector_set_multiple_variables, + new step\flow_set_multiple_variables, ]; } diff --git a/run.php b/run.php index c1d8fae4..d3edb790 100644 --- a/run.php +++ b/run.php @@ -44,8 +44,8 @@ function tool_dataflows_mtrace_wrapper($message, $eol) { // Mark up errors.. if (preg_match('/error:/im', $message)) { $class = 'bg-danger text-white'; - } else if (preg_match('/warn:/im', $message)) { - $class = 'bg-warning'; + } else if (preg_match('/warning:/im', $message)) { + $class = 'text-warning'; } else if (preg_match('/notice:/im', $message)) { $class = 'bold text-primary'; } else if (preg_match('/info:/im', $message)) { diff --git a/version.php b/version.php index aeecfecd..af03ce66 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023101600; -$plugin->release = 2023101600; +$plugin->version = 2023122200; +$plugin->release = 2023122200; $plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0). // TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later. $plugin->component = 'tool_dataflows';