From 9e6527543334decafee5b0282ce7e28e2b6bf313 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Tue, 19 Dec 2023 12:15:26 +1100 Subject: [PATCH 01/16] feat: add connector SQL step --- classes/local/step/connector_sql.php | 29 ++++++ classes/local/step/flow_sql.php | 127 --------------------------- classes/local/step/sql_trait.php | 113 ++++++++++++++++++++++++ lang/en/tool_dataflows.php | 1 + lib.php | 2 + 5 files changed, 145 insertions(+), 127 deletions(-) create mode 100644 classes/local/step/connector_sql.php diff --git a/classes/local/step/connector_sql.php b/classes/local/step/connector_sql.php new file mode 100644 index 00000000..0cb01e5d --- /dev/null +++ b/classes/local/step/connector_sql.php @@ -0,0 +1,29 @@ +. + +namespace tool_dataflows\local\step; + +/** + * SQL connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_sql extends connector_step { + use sql_trait; +} diff --git a/classes/local/step/flow_sql.php b/classes/local/step/flow_sql.php index 0bfff0ca..1e91ec76 100644 --- a/classes/local/step/flow_sql.php +++ b/classes/local/step/flow_sql.php @@ -27,131 +27,4 @@ class flow_sql extends flow_step { use sql_trait; - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'sql' => ['type' => PARAM_TEXT, 'required' => true], - ]; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - // SQL example and inputs. - $sqlexample = " - SELECT id, username - FROM {user} - WHERE id > \${{steps.xyz.var.number}} - ORDER BY id ASC - LIMIT 10"; - $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); - $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), - ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); - $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); - } - - /** - * Allow steps to setup the form depending on current values. - * - * This method is called after definition(), data submission and set_data(). - * All form setup that is dependent on form values should go in here. - * - * @param \MoodleQuickForm $mform - * @param \stdClass $data - */ - public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { - // Validate the data. - $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); - - // Get the element. - $element = $mform->getElement('config_sql'); - - // Update the element height based on min/max settings, but preserve - // other existing rules. - $attributes = $element->getAttributes(); - - // Set the rows at a minimum to the predefined amount in - // form_add_custom_inputs, and expand as content grows up to a maximum. - $attributes['rows'] = min( - $attributes['max_rows'], - max($attributes['rows'], $sqllinecount) - ); - $element->setAttributes($attributes); - } - - /** - * Execute configured query - * - * @param mixed $input - * @return mixed - * @throws \dml_read_exception when the SQL is not valid. - */ - public function execute($input = null) { - global $DB; - - // Construct the query. - $variables = $this->get_variables(); - $config = $variables->get_raw('config'); - [$sql, $params] = $this->evaluate_expressions($config->sql); - - // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. - // This is not used for security, just to route the request via the correct pathway for readonly databases. - $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; - $matches = []; - preg_match($pattern, $sql, $matches); - - // Matches[0] contains the match. Fallthrough to default on no match. - $token = $matches[0] ?? ''; - $emptydefault = new \stdClass(); - - switch(strtoupper($token)) { - case 'SELECT': - // Execute the query using get_records instead of get_record. - // This is so we can expose the number of records returned which - // can then be used by the dataflow in for e.g. a switch statement. - $records = $DB->get_records_sql($sql, $params); - - $variables->set('count', count($records)); - $invalidnum = ($records === false || count($records) !== 1); - $data = $invalidnum ? $emptydefault : array_pop($records); - $variables->set('data', $data); - break; - default: - // Default to execute. - $success = $DB->execute($sql, $params); - - // We can't really do anything with the response except check for success. - $variables->set('count', (int) $success); - $variables->set('data', $emptydefault); - break; - } - - return $input; - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (empty($config->sql)) { - $errors['config_sql'] = get_string('config_field_missing', 'tool_dataflows', 'sql', true); - } - - return empty($errors) ? true : $errors; - } } diff --git a/classes/local/step/sql_trait.php b/classes/local/step/sql_trait.php index 5b1157cc..05a5234a 100644 --- a/classes/local/step/sql_trait.php +++ b/classes/local/step/sql_trait.php @@ -167,4 +167,117 @@ public function validate_config($config) { } return empty($errors) ? true : $errors; } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'sql' => ['type' => PARAM_TEXT, 'required' => true], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + // SQL example and inputs. + $sqlexample = " + SELECT id, username + FROM {user} + WHERE id > \${{steps.xyz.var.number}} + ORDER BY id ASC + LIMIT 10"; + $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); + $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), + ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); + $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); + } + + /** + * Allow steps to setup the form depending on current values. + * + * This method is called after definition(), data submission and set_data(). + * All form setup that is dependent on form values should go in here. + * + * @param \MoodleQuickForm $mform + * @param \stdClass $data + */ + public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { + // Validate the data. + $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); + + // Get the element. + $element = $mform->getElement('config_sql'); + + // Update the element height based on min/max settings, but preserve + // other existing rules. + $attributes = $element->getAttributes(); + + // Set the rows at a minimum to the predefined amount in + // form_add_custom_inputs, and expand as content grows up to a maximum. + $attributes['rows'] = min( + $attributes['max_rows'], + max($attributes['rows'], $sqllinecount) + ); + $element->setAttributes($attributes); + } + + /** + * Execute configured query + * + * @param mixed $input + * @return mixed + * @throws \dml_read_exception when the SQL is not valid. + */ + public function execute($input = null) { + global $DB; + + // Construct the query. + $variables = $this->get_variables(); + $config = $variables->get_raw('config'); + [$sql, $params] = $this->evaluate_expressions($config->sql); + + // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. + // This is not used for security, just to route the request via the correct pathway for readonly databases. + $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; + $matches = []; + preg_match($pattern, $sql, $matches); + + // Matches[0] contains the match. Fallthrough to default on no match. + $token = $matches[0] ?? ''; + $emptydefault = new \stdClass(); + + switch(strtoupper($token)) { + case 'SELECT': + // Execute the query using get_records instead of get_record. + // This is so we can expose the number of records returned which + // can then be used by the dataflow in for e.g. a switch statement. + $records = $DB->get_records_sql($sql, $params); + + $variables->set('count', count($records)); + $invalidnum = ($records === false || count($records) !== 1); + $data = $invalidnum ? $emptydefault : array_pop($records); + $variables->set('data', $data); + break; + default: + // Default to execute. + $success = $DB->execute($sql, $params); + + // We can't really do anything with the response except check for success. + $variables->set('count', (int) $success); + $variables->set('data', $emptydefault); + break; + } + + return $input; + } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 97ec6645..135daac2 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -179,6 +179,7 @@ $string['step_name_writer_stream'] = 'Stream writer'; $string['step_name_trigger_event'] = 'Moodle event'; $string['step_name_flow_sql'] = 'SQL'; +$string['step_name_connector_sql'] = 'SQL'; // Step (type) groups. $string['stepgrouptriggers'] = 'Triggers'; diff --git a/lib.php b/lib.php index 0f9404e2..af08939c 100644 --- a/lib.php +++ b/lib.php @@ -68,6 +68,8 @@ function tool_dataflows_step_types() { new step\connector_s3, new step\connector_set_variable, new step\connector_sftp, + new step\connector_sql, + new step\connector_update_user, new step\connector_sftp_directory_file_list, new step\connector_sns_notify, new step\connector_wait, From 5f77d7ed4d0fbf88d87828ed365586edfcc06882 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Tue, 19 Dec 2023 14:25:33 +1100 Subject: [PATCH 02/16] fix: allow values to be corrected in bad states Instead of allowing the state to be not recoverable, this will instead fail to render the available variables section only. --- classes/form/step_form.php | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/classes/form/step_form.php b/classes/form/step_form.php index 5eec7d04..7e3af96e 100644 --- a/classes/form/step_form.php +++ b/classes/form/step_form.php @@ -107,11 +107,22 @@ public function definition() { $select->setMultiple(true); $dataflow = new dataflow($dataflowid); - $varroot = $dataflow->get_variables_root(); - // List all the available fields available for configuration, in dot syntax. - $mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'), - $this->prepare_available_fields($varroot->get())); + try { + $varroot = $dataflow->get_variables_root(); + + // List all the available fields available for configuration, in dot syntax. + $mform->addElement( + 'static', + 'fields', + get_string('available_fields', 'tool_dataflows'), + $this->prepare_available_fields($varroot->get()) + ); + } catch (\Throwable $e) { + global $OUTPUT; + $errtext = $OUTPUT->notification($e->getMessage()); + $mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext); + } // Check and set custom form inputs if required. Defaulting to a // textarea config input for those not yet configured. From 1a357a1db5fe2e9d1f25a95ea389f1b7242cc53b Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Tue, 19 Dec 2023 14:27:13 +1100 Subject: [PATCH 03/16] feat: add set multiple variables step type Similar to set_variable but this allows multiple values to be set on an existing or new path. This simplifies initialisation of multiple variables per instance run, such as counters used for reporting. --- .../step/connector_set_multiple_variables.php | 35 ++++++ .../step/flow_set_multiple_variables.php | 35 ++++++ .../step/set_multiple_variables_trait.php | 101 ++++++++++++++++++ lang/en/tool_dataflows.php | 8 ++ lib.php | 2 + 5 files changed, 181 insertions(+) create mode 100644 classes/local/step/connector_set_multiple_variables.php create mode 100644 classes/local/step/flow_set_multiple_variables.php create mode 100644 classes/local/step/set_multiple_variables_trait.php diff --git a/classes/local/step/connector_set_multiple_variables.php b/classes/local/step/connector_set_multiple_variables.php new file mode 100644 index 00000000..b609490b --- /dev/null +++ b/classes/local/step/connector_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_set_multiple_variables extends connector_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/flow_set_multiple_variables.php b/classes/local/step/flow_set_multiple_variables.php new file mode 100644 index 00000000..1115240e --- /dev/null +++ b/classes/local/step/flow_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables flow step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_set_multiple_variables extends flow_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/set_multiple_variables_trait.php b/classes/local/step/set_multiple_variables_trait.php new file mode 100644 index 00000000..6d2ea232 --- /dev/null +++ b/classes/local/step/set_multiple_variables_trait.php @@ -0,0 +1,101 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\local\variables\var_root; +use tool_dataflows\local\variables\var_object_visible; + +/** + * Set multiple variables trait + * + * Similar to the single approach, except it allows multiple to be set in a + * single step. This is great for initialising counters, and initial variables + * that need to be reset every run, but might change during. + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait set_multiple_variables_trait { + use set_variable_trait; + + /** + * Executes the step, fetching the config and actioning the step. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + + $stepvars = $this->get_variables(); + $config = $stepvars->get('config'); + $rootvars = $this->get_variables_root(); + $this->run($rootvars, $config->field, $config->values); + + return $input; + } + + /** + * Main handler of this step, split out to make it easier to test. + * + * @param var_root $varobject + * @param string $field + * @param mixed $values + */ + public function run(var_root $varobject, string $field, $values) { + // Do nothing if the value has not changed. + $currentvalue = $varobject->get($field); + if ($currentvalue === $values) { + return; + } + + // Set the value in the variable tree. + $varobject->set($field, $values); + $this->log->info("Set '{field}' as '{values}'", ['field' => $field, 'values' => json_encode($values)]); + + // We do not persist the value if it is a dry run. + if ($this->is_dry_run()) { + return; + } + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'field' => ['type' => PARAM_TEXT, 'required' => true], + 'values' => ['type' => PARAM_TEXT, 'required' => true, 'yaml' => true], + ]; + } + + /** + * Custom form inputs + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_field', get_string('set_multiple_variables:field', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:field_help', 'tool_dataflows')); + + $mform->addElement('textarea', 'config_values', get_string('set_multiple_variables:values', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:values_help', 'tool_dataflows')); + } +} diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 135daac2..dc969e47 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -145,6 +145,7 @@ $string['step_name_connector_remove_file'] = 'Remove file'; $string['step_name_connector_s3'] = 'S3 file copy'; $string['step_name_connector_set_variable'] = 'Set variable'; +$string['step_name_connector_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_connector_sftp'] = 'SFTP file copy'; $string['step_name_connector_sns_notify'] = 'AWS-SNS Notification'; $string['step_name_connector_wait'] = 'Wait'; @@ -159,6 +160,7 @@ $string['step_name_flow_hash_file'] = 'Hash file'; $string['step_name_flow_remove_file'] = 'Remove file'; $string['step_name_flow_set_variable'] = 'Set variable'; +$string['step_name_flow_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_flow_logic_join'] = 'Join'; $string['step_name_flow_logic_switch'] = 'Switch'; $string['step_name_flow_noop'] = 'No-op'; @@ -589,6 +591,12 @@ $string['set_variable:value'] = 'Value'; $string['set_variable:value_help'] = 'The value could be a number, text, or an expression. For example: ${{ record.id }}.'; +// Set multiple variables step. +$string['set_multiple_variables:field'] = 'Field'; +$string['set_multiple_variables:field_help'] = 'Defines the path to the field you would like to set the value(s). For example: dataflow.vars.counter.'; +$string['set_multiple_variables:values'] = 'Values'; +$string['set_multiple_variables:values_help'] = 'A list of fields/keys and values, in YAML format.'; + // Event trigger. $string['trigger_event:policy:immediate'] = 'Run immediately'; $string['trigger_event:policy:adhoc'] = 'Run ASAP in individual tasks in parallel'; diff --git a/lib.php b/lib.php index af08939c..c63c4d16 100644 --- a/lib.php +++ b/lib.php @@ -105,6 +105,8 @@ function tool_dataflows_step_types() { new step\trigger_webservice, new step\writer_debugging, new step\writer_stream, + new step\connector_set_multiple_variables, + new step\flow_set_multiple_variables, ]; } From 328865124526b45308fcc9d25876dbf419f84825 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Wed, 20 Dec 2023 13:14:19 +1100 Subject: [PATCH 04/16] fix: warning logs now display and are more legible - The background with white text was difficult to read (low contrast) - The warning regex previously in place was incorrect --- run.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.php b/run.php index c1d8fae4..d3edb790 100644 --- a/run.php +++ b/run.php @@ -44,8 +44,8 @@ function tool_dataflows_mtrace_wrapper($message, $eol) { // Mark up errors.. if (preg_match('/error:/im', $message)) { $class = 'bg-danger text-white'; - } else if (preg_match('/warn:/im', $message)) { - $class = 'bg-warning'; + } else if (preg_match('/warning:/im', $message)) { + $class = 'text-warning'; } else if (preg_match('/notice:/im', $message)) { $class = 'bold text-primary'; } else if (preg_match('/info:/im', $message)) { From c5eb128d108d1cc287ae137e630b373caf1702cd Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Wed, 20 Dec 2023 13:14:52 +1100 Subject: [PATCH 05/16] fix: add copy file error handling for directory creation --- classes/local/step/copy_file_trait.php | 14 +++++++++----- lang/en/tool_dataflows.php | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/classes/local/step/copy_file_trait.php b/classes/local/step/copy_file_trait.php index 30a081c5..05cc866b 100644 --- a/classes/local/step/copy_file_trait.php +++ b/classes/local/step/copy_file_trait.php @@ -72,7 +72,9 @@ public function execute($input = null) { $todirectory = dirname($to); if (!file_exists($todirectory)) { $this->log("Creating a directory at {$todirectory}"); - mkdir($todirectory, $CFG->directorypermissions, true); + if (!mkdir($todirectory, $CFG->directorypermissions, true)) { + throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory); + } } // Attempt to copy the file to the destination. @@ -112,10 +114,12 @@ public function execute($input = null) { private function copy(string $from, string $to) { $this->log("Copying $from to $to"); if (!copy($from, $to)) { - throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [ - 'from' => $from, - 'to' => $to, - ]); + throw new \moodle_exception( + 'flow_copy_file:copy_failed', + 'tool_dataflows', + '', + (object) ['from' => $from, 'to' => $to] + ); } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index dc969e47..6a7d1d0b 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -549,6 +549,7 @@ $string['flow_copy_file:from'] = 'From'; $string['flow_copy_file:to'] = 'To'; $string['flow_copy_file:copy_failed'] = 'Failed to copy {$a->from} to {$a->to}'; +$string['flow_copy_file:mkdir_failed'] = 'Failed to create directory at {$a}. Please check permissions and try again.'; // Directory file count. $string['connector_directory_file_count:path'] = 'Path to directory'; From 34175eefbf12529c0f65690589e81a279b5db8b2 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Wed, 20 Dec 2023 13:15:40 +1100 Subject: [PATCH 06/16] feat: run name and id are now available as expressable vars Also tweaked comment on string format of run based output logs. --- classes/local/execution/engine.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php index ae845cbd..f2ac9377 100644 --- a/classes/local/execution/engine.php +++ b/classes/local/execution/engine.php @@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat // Find the flow blocks. $this->create_flow_caps(); + + // Make the runid available to the flow. + if (!$this->isdryrun) { + $variables = $this->get_variables(); + $variables->set('run.name', $this->run->name); + $variables->set('run.id', $this->run->id); + } } /** @@ -785,7 +792,8 @@ private function setup_logging() { } // Dataflow run logger. - // e.g. '[dataroot]/tool_dataflows/3/21.log' as the path. + // Type: FILE_PER_RUN + // e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path. if (isset($loghandlers[log_handler::FILE_PER_RUN])) { $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . From 75d7e1d131a97a2b83450c5acaa904dad0cf1ed3 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 11:02:47 +1100 Subject: [PATCH 07/16] fix: SQL trait handling of integer values and improve error message --- classes/local/step/sql_trait.php | 4 ++-- lang/en/tool_dataflows.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/local/step/sql_trait.php b/classes/local/step/sql_trait.php index 05a5234a..e3c13762 100644 --- a/classes/local/step/sql_trait.php +++ b/classes/local/step/sql_trait.php @@ -75,8 +75,8 @@ private function evaluate_expressions(string $sql) { [$hasexpression] = $parser->has_expression($el); $max--; } - if (!in_array(gettype($el), ['string', 'int'])) { - throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows'); + if (!in_array(gettype($el), ['string', 'int', 'integer'])) { + throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows', '', gettype($el)); } return $el; }, $expressions); diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 6a7d1d0b..636d0814 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -353,7 +353,7 @@ $string['writer_csv:fail_to_encode'] = 'Failed to encode CSV.'; // SQL trait. -$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int).'; +$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int), found {$a}'; $string['sql_trait:variable_not_valid_in_position_replacement_text'] = "Invalid expression \${{ {\$a->expression} }} as `{\$a->expressionpath}` could not be resolved at line {\$a->line} character {\$a->column} in:\n{\$a->sql}"; // phpcs:disable moodle.Strings.ForbiddenStrings.Found // Reader SQL. From 49d2268d49c5accba1f1c019e1722c68d6be118b Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 11:03:17 +1100 Subject: [PATCH 08/16] fix: update email flow step to support it as a final step --- classes/local/step/flow_email.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/classes/local/step/flow_email.php b/classes/local/step/flow_email.php index 7090dfdd..d35d1510 100644 --- a/classes/local/step/flow_email.php +++ b/classes/local/step/flow_email.php @@ -28,6 +28,12 @@ */ class flow_email extends flow_step { + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max) */ + protected $outputconnectors = [0, 1]; + /** @var bool whether or not this step type (potentially) contains a side effect or not */ protected $hassideeffect = true; From d971728c760811f0e670a942c926c9ff02c0e85f Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 11:04:19 +1100 Subject: [PATCH 09/16] fix: reader csv error to be more correct for header/fields count alignment It is more correct to expect the number of fields to match the number of headers, not the other way around necessarily. --- lang/en/tool_dataflows.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 636d0814..4bf171bf 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -373,7 +373,7 @@ $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.'; $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers'; $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.'; -$string['reader_csv:header_field_count_mismatch'] = 'Number of headers ({$a->numheaders}) should match number of fields ({$a->numfields})'; +$string['reader_csv:header_field_count_mismatch'] = 'Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; // Reader JSON. $string['reader_json:arrayexpression_help'] = 'Nested array to extract from JSON. For example, {$a->expression} will return the users array from the following JSON (If empty it is assumed the starting point of the JSON file is an array):{$a->jsonexample}'; From 929c393526e764cad62e3e78f4bfd64955449089 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 11:49:45 +1100 Subject: [PATCH 10/16] fix: include row number in csv reader error --- classes/local/step/reader_csv.php | 3 +++ lang/en/tool_dataflows.php | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/classes/local/step/reader_csv.php b/classes/local/step/reader_csv.php index c5f8e7aa..b9437803 100644 --- a/classes/local/step/reader_csv.php +++ b/classes/local/step/reader_csv.php @@ -92,12 +92,15 @@ public function csv_contents_generator() { // Convert header string to an actual headers array. $headers = str_getcsv($strheaders, $delimiter); $numheaders = count($headers); + $rownumber = 1; // First row is always headers. while (($data = fgetcsv($handle, $maxlinelength, $delimiter)) !== false) { + $rownumber++; $numfields = count($data); if ($numfields !== $numheaders) { throw new \moodle_exception('reader_csv:header_field_count_mismatch', 'tool_dataflows', '', (object) [ 'numfields' => $numfields, 'numheaders' => $numheaders, + 'rownumber' => $rownumber, ], json_encode([ 'fields' => $data, 'headers' => $headers, diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 4bf171bf..d7dea62d 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -373,7 +373,7 @@ $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.'; $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers'; $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.'; -$string['reader_csv:header_field_count_mismatch'] = 'Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; +$string['reader_csv:header_field_count_mismatch'] = 'Row #{$a->rownumber}: Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; // Reader JSON. $string['reader_json:arrayexpression_help'] = 'Nested array to extract from JSON. For example, {$a->expression} will return the users array from the following JSON (If empty it is assumed the starting point of the JSON file is an array):{$a->jsonexample}'; From 97832c8678952b7c7dcfb5591c4555e62df601e3 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 15:11:39 +1100 Subject: [PATCH 11/16] perf: cache sftp object per identity to speed up subsequent actions --- classes/local/step/sftp_trait.php | 69 ++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/classes/local/step/sftp_trait.php b/classes/local/step/sftp_trait.php index 1304dea4..0c61c53b 100644 --- a/classes/local/step/sftp_trait.php +++ b/classes/local/step/sftp_trait.php @@ -35,10 +35,13 @@ trait sftp_trait { /** Shorthand sftp scheme for use in config. */ - static protected $sftpprefix = 'sftp'; + protected static $sftpprefix = 'sftp'; /** Default port to connect to. */ - static protected $defaultport = 22; + protected static $defaultport = 22; + + /** Array of SFTP objects to use for performance reasons. */ + protected $sftp = []; /** * Returns whether or not the step configured, has a side effect. @@ -93,17 +96,14 @@ public function form_add_core_inputs(\MoodleQuickForm &$mform) { $mform->addElement('text', 'config_port', get_string('connector_sftp:port', 'tool_dataflows')); $mform->setDefault('config_port', self::$defaultport); $mform->addElement('text', 'config_hostpubkey', get_string('connector_sftp:hostpubkey', 'tool_dataflows')); - $mform->addElement('static', 'config_hostpubkey_desc', '', - get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_hostpubkey_desc', '', get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); $mform->addElement('text', 'config_username', get_string('username')); $mform->addElement('passwordunmask', 'config_password', get_string('password')); - $mform->addElement('static', 'config_password_desc', '', - get_string('connector_sftp:password_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_password_desc', '', get_string('connector_sftp:password_desc', 'tool_dataflows')); $mform->addElement('text', 'config_privkeyfile', get_string('connector_sftp:privkeyfile', 'tool_dataflows')); - $mform->addElement('static', 'config_keyfile_desc', '', - get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_keyfile_desc', '', get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); } /** @@ -118,13 +118,19 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform, $behaviour = 'c if ($behaviour === 'copy') { $mform->addElement('text', 'config_source', get_string('connector_sftp:source', 'tool_dataflows')); $mform->addElement('static', 'config_source_desc', '', get_string('connector_sftp:source_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); $mform->addElement('text', 'config_target', get_string('connector_sftp:target', 'tool_dataflows')); $mform->addElement('static', 'config_target_desc', '', get_string('connector_sftp:target_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); } } @@ -247,11 +253,8 @@ public function execute($input = null) { $stepvars = $this->get_variables(); $config = $stepvars->get('config'); - $this->log->debug("Connecting to {$config->host}:{$config->port}"); - // At this point we need to disconnect once we are finished. try { - // Skip if it is a dry run. if ($this->is_dry_run() && $this->has_side_effect()) { return $input; @@ -278,7 +281,7 @@ public function execute($input = null) { // Upload to remote. $this->upload($sftp, $sourcepath, $targetpath); - } finally { + } catch (\Throwable $e) { if (isset($sftp)) { $sftp->disconnect(); } @@ -287,6 +290,28 @@ public function execute($input = null) { return $input; } + /** + * Hook function that gets called when an engine step has been aborted. + */ + public function on_abort() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + + /** + * Hook function that gets called when an engine step has been finalised. + */ + public function on_finalise() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + /** * Checks and loads the appropriate key, based on config * @@ -408,6 +433,14 @@ public function resolve_path(string $pathname): string { * @return SFTP */ private function init_sftp($config): SFTP { + // Use existing cached SFTP object if available. + $cachekey = implode('|', [$config->host, $config->port, $config->username]); + if (isset($this->sftp[$cachekey])) { + return $this->sftp[$cachekey]; + } + + // Create and connect to SFTP. + $this->log->debug("Connecting to {$config->host}:{$config->port}"); $sftp = new SFTP($config->host, $config->port); $this->check_public_host_key($sftp, $config->hostpubkey); @@ -417,6 +450,10 @@ private function init_sftp($config): SFTP { } $sftp->enableDatePreservation(); + + // Cache sftp since it takes a while to attempt initial connection. + $this->sftp[$cachekey] = $sftp; + return $sftp; } } From 9555886e1f581091ff18d392c54a4d964a07ca1a Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 21 Dec 2023 15:30:30 +1100 Subject: [PATCH 12/16] feat: add an option to continue processing the csv file on known parse issues The only one is currently a count mismatch on header and the row fields --- classes/local/step/reader_csv.php | 31 ++++++++++++++++++++++++++++++- lang/en/tool_dataflows.php | 2 ++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/classes/local/step/reader_csv.php b/classes/local/step/reader_csv.php index b9437803..1de5e7f0 100644 --- a/classes/local/step/reader_csv.php +++ b/classes/local/step/reader_csv.php @@ -43,6 +43,7 @@ public static function form_define_fields(): array { 'path' => ['type' => PARAM_TEXT, 'required' => true], 'headers' => ['type' => PARAM_TEXT], 'overwriteheaders' => ['type' => PARAM_BOOL], + 'continueonerror' => ['type' => PARAM_BOOL], 'delimiter' => ['type' => PARAM_TEXT], ]; } @@ -61,9 +62,11 @@ public function get_iterator(): iterator { */ public function csv_contents_generator() { $maxlinelength = 1000; - $config = $this->get_variables()->get('config'); + $variables = $this->get_variables(); + $config = $variables->get('config'); $strheaders = $config->headers; $overwriteheaders = !empty($config->overwriteheaders); + $continueonerror = !empty($config->continueonerror); $delimiter = $config->delimiter ?: self::DEFAULT_DELIMETER; $path = $this->enginestep->engine->resolve_path($config->path); @@ -93,10 +96,30 @@ public function csv_contents_generator() { $headers = str_getcsv($strheaders, $delimiter); $numheaders = count($headers); $rownumber = 1; // First row is always headers. + $errors = ['header_field_count_mismatch' => 0]; while (($data = fgetcsv($handle, $maxlinelength, $delimiter)) !== false) { $rownumber++; $numfields = count($data); if ($numfields !== $numheaders) { + // Continue on (parse) error. + if ($continueonerror) { + $errors['header_field_count_mismatch'] += 1; + $this->log->error( + get_string('reader_csv:header_field_count_mismatch', 'tool_dataflows', (object) [ + 'numfields' => $numfields, + 'numheaders' => $numheaders, + 'rownumber' => $rownumber, + ]), + [ + 'fields' => $data, + 'headers' => $headers, + ] + ); + + continue; + } + + // Throw exception on error. throw new \moodle_exception('reader_csv:header_field_count_mismatch', 'tool_dataflows', '', (object) [ 'numfields' => $numfields, 'numheaders' => $numheaders, @@ -112,6 +135,8 @@ public function csv_contents_generator() { } finally { fclose($handle); } + + $variables->set('errors', (object) $errors); } /** @@ -153,6 +178,10 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) { $mform->hideIf('config_overwriteheaders', 'config_headers', 'eq', ''); $mform->disabledIf('config_overwriteheaders', 'config_headers', 'eq', ''); + // Used when we want to replace the headers (or overwrite them) using the ones we supplied instead. Defaults to off. + $mform->addElement('checkbox', 'config_continueonerror', get_string('reader_csv:continueonerror', 'tool_dataflows'), + get_string('reader_csv:continueonerror_help', 'tool_dataflows')); + // Delimiter. $mform->addElement( 'text', diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index d7dea62d..111f0b7b 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -373,6 +373,8 @@ $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.'; $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers'; $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.'; +$string['reader_csv:continueonerror'] = 'Continue on parsing errors'; +$string['reader_csv:continueonerror_help'] = 'If checked, the step will continue reading the next row of data if there are parsing errors on the current.'; $string['reader_csv:header_field_count_mismatch'] = 'Row #{$a->rownumber}: Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; // Reader JSON. From db9cc3217483dd616af463111ee618170acad806 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 9 Nov 2023 16:40:41 +1100 Subject: [PATCH 13/16] feat: add find step - returns a match from a list --- classes/local/step/connector_find.php | 92 +++++++++++++++++++++++++++ lang/en/tool_dataflows.php | 8 +++ lib.php | 2 + 3 files changed, 102 insertions(+) create mode 100644 classes/local/step/connector_find.php diff --git a/classes/local/step/connector_find.php b/classes/local/step/connector_find.php new file mode 100644 index 00000000..1a51c6cd --- /dev/null +++ b/classes/local/step/connector_find.php @@ -0,0 +1,92 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Find (a record from a collection) + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2022 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_find extends connector_step { + + /** + * Returns whether the step has a side effect. + * + * @return bool whether or not this step has a side effect + * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) + */ + public function has_side_effect(): bool { + return false; + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'collection' => ['type' => PARAM_TEXT, 'required' => true], + 'condition' => ['type' => PARAM_TEXT, 'required' => true], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_collection', get_string('find:collection', 'tool_dataflows')); + $mform->addElement('static', 'config_collection_help', '', get_string('find:collection_help', 'tool_dataflows')); + $mform->addElement('text', 'config_condition', get_string('find:condition', 'tool_dataflows')); + $mform->addElement('static', 'config_condition_help', '', get_string('find:condition_help', 'tool_dataflows')); + } + + /** + * Find based on the condition, and set the matching record given a record. + * + * If the message is empty, it will not log anything, which is useful for conditional logging. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + $variables = $this->get_variables(); + $config = $variables->get('config'); + $collection = $config->collection; + + foreach ($collection as $item) { + $variables->set('record', $item); + $condition = $variables->evaluate('${{'.$config->condition.'}}'); + if ($condition) { + $input = $item; + break; + } + } + $variables->set('match', $input); + + return $input; + } +} diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 111f0b7b..a6a99954 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -634,3 +634,11 @@ $string['log:level'] = 'Log Level'; $string['log:level_help'] = 'Choose between 8 logging levels. Each level conveys a different meaning, and it is best practice to choose the most appropriate one for the log message.'; $string['log:message'] = 'Message'; + +// Find step. +$string['step_name_connector_find'] = 'Find'; +$string['find:collection'] = 'Collection'; +$string['find:collection_help'] = 'A reference to a collection of items.'; +$string['find:condition'] = 'Condition'; +$string['find:condition_help'] = 'The expression checked against each item in the referenced collection. For each item in the collection, it can be referenced as "item.field"'; + diff --git a/lib.php b/lib.php index c63c4d16..f5bf130a 100644 --- a/lib.php +++ b/lib.php @@ -60,6 +60,7 @@ function tool_dataflows_step_types() { new step\connector_email, new step\connector_file_exists, new step\connector_file_put_content, + new step\connector_find, new step\connector_gpg, new step\connector_hash_file, new step\connector_log, @@ -94,6 +95,7 @@ function tool_dataflows_step_types() { new step\flow_transformer_alter, new step\flow_transformer_filter, new step\flow_transformer_regex, + new step\flow_update_user, new step\flow_web_service, new step\reader_csv, new step\reader_directory_file_list, From 404c79f0b5408bc4a463465fd87ddfd73a19d33d Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Thu, 9 Nov 2023 16:58:09 +1100 Subject: [PATCH 14/16] feat: add update user step Easily update user profile fields given a user id and a yaml formatted list of fields to update --- classes/local/step/connector_update_user.php | 35 ++++++++ classes/local/step/flow_update_user.php | 35 ++++++++ classes/local/step/update_user_trait.php | 93 ++++++++++++++++++++ lang/en/tool_dataflows.php | 7 ++ 4 files changed, 170 insertions(+) create mode 100644 classes/local/step/connector_update_user.php create mode 100644 classes/local/step/flow_update_user.php create mode 100644 classes/local/step/update_user_trait.php diff --git a/classes/local/step/connector_update_user.php b/classes/local/step/connector_update_user.php new file mode 100644 index 00000000..01e569df --- /dev/null +++ b/classes/local/step/connector_update_user.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_update_user extends connector_step { + use update_user_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/flow_update_user.php b/classes/local/step/flow_update_user.php new file mode 100644 index 00000000..023b9195 --- /dev/null +++ b/classes/local/step/flow_update_user.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user flow step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_update_user extends flow_step { + use update_user_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/update_user_trait.php b/classes/local/step/update_user_trait.php new file mode 100644 index 00000000..e5204263 --- /dev/null +++ b/classes/local/step/update_user_trait.php @@ -0,0 +1,93 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Update user using core api + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait update_user_trait { + + /** + * Returns whether or not the step configured, has a side effect. + * + * @return bool whether or not this step has a side effect + * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) + */ + public function has_side_effect(): bool { + return true; + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'userid' => ['type' => PARAM_TEXT, 'required' => true], + 'fields' => ['type' => PARAM_TEXT, 'required' => true, 'yaml' => true], + ]; + } + + /** + * Custom elements for editing the connector. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_userid', get_string('update_user:userid', 'tool_dataflows')); + $mform->addElement('static', 'config_userid_help', '', get_string('update_user:userid_help', 'tool_dataflows')); + + $mform->addElement( + 'textarea', + 'config_fields', + get_string('update_user:fields', 'tool_dataflows'), + ['cols' => 60, 'rows' => 5] + ); + $mform->addElement('static', 'config_fields_help', '', get_string('update_user:fields_help', 'tool_dataflows')); + } + + /** + * Main execution method which updates the user's details. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + $variables = $this->get_variables(); + $config = $variables->get('config'); + $userobject = (object) array_merge( + ['id' => $config->userid], + (array) $config->fields + ); + + // Some fields will not be updatable (intentionally). + unset($userobject->password); + unset($userobject->auth); + + // Update user fields using core api. + user_update_user($userobject, false, false); + profile_save_data($userobject); + + return $input; + } +} diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index a6a99954..a39d941a 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -642,3 +642,10 @@ $string['find:condition'] = 'Condition'; $string['find:condition_help'] = 'The expression checked against each item in the referenced collection. For each item in the collection, it can be referenced as "item.field"'; +// Update user step. +$string['step_name_connector_update_user'] = 'Update user'; +$string['step_name_flow_update_user'] = 'Update user'; +$string['update_user:userid'] = 'User ID'; +$string['update_user:userid_help'] = 'The internal ID for this user.'; +$string['update_user:fields'] = 'Fields'; +$string['update_user:fields_help'] = 'Fields set here correspond to the fields available in the underlying user table. Custom user profile fields, must be prefixed with "profile_field_" followed by the shortname. Not all fields can be updated, such as password and auth'; From a97ff72cf2f3c847c81ad2ab7c421c89b4960aba Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Tue, 14 Nov 2023 10:38:11 +1100 Subject: [PATCH 15/16] fix!: adjust filename of log files to include datetime - Add step in upgrade to update existing log files --- classes/local/execution/engine.php | 17 +++++-- classes/local/step/flow_transformer_regex.php | 1 + db/upgrade.php | 44 +++++++++++++++++++ lang/en/tool_dataflows.php | 4 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php index f2ac9377..1494178e 100644 --- a/classes/local/execution/engine.php +++ b/classes/local/execution/engine.php @@ -753,6 +753,12 @@ private function setup_logging() { $channel .= '/' . $this->run->name; } + // Set the starting time as 'now'. + $now = microtime(true); + [, $decimal] = explode('.', $now); + $decimal = substr($decimal, 0, 3); // Only use the first 3 digits after the decimal point. + $rundateformat = date("Ymd_His$decimal", $now); + // Each channel represents a specific way of writing log information. $log = new Logger($channel); @@ -797,8 +803,8 @@ private function setup_logging() { if (isset($loghandlers[log_handler::FILE_PER_RUN])) { $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . - $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name - . '.log'; + $this->dataflow->id . DIRECTORY_SEPARATOR . + $rundateformat . '_' . $this->run->name . '.log'; $streamhandler = new StreamHandler($dataflowrunlogpath, Logger::DEBUG); $streamhandler->setFormatter($lineformatter); @@ -806,13 +812,18 @@ private function setup_logging() { } // General dataflow logger (rotates daily to prevent big single log file). - // e.g. '[dataroot]/tool_dataflows/3-2006-01-02.log' as the path. + // Type: FILE_PER_DATAFLOW + // e.g. '[dataroot]/tool_dataflows/20060102-3.log' as the path. if (isset($loghandlers[log_handler::FILE_PER_DATAFLOW])) { $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . $this->dataflow->id . '.log'; $rotatingfilehandler = new RotatingFileHandler($dataflowlogpath, 0, Logger::DEBUG); + $dateformat = 'Ymd'; + $filenameformat = '{date}_{filename}'; + $rotatingfilehandler->setFilenameFormat($filenameformat, $dateformat); + $rotatingfilehandler->setFormatter($lineformatter); $log->pushHandler($rotatingfilehandler); } diff --git a/classes/local/step/flow_transformer_regex.php b/classes/local/step/flow_transformer_regex.php index d4261157..1f50c2b0 100644 --- a/classes/local/step/flow_transformer_regex.php +++ b/classes/local/step/flow_transformer_regex.php @@ -62,6 +62,7 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) { get_string('flow_transformer_regex:pattern', 'tool_dataflows'), [ 'placeholder' => "/[abc]/", + 'size' => '60', ] ); $mform->addElement( diff --git a/db/upgrade.php b/db/upgrade.php index 90e076c0..29a56016 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -287,5 +287,49 @@ function xmldb_tool_dataflows_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2023072100, 'tool', 'dataflows'); } + // Move log files that exist across to new format. Breaking change if any + // dataflows implement logic based on these files based on filename format. + if ($oldversion < 2023110901) { + $path = '*.log'; + $pattern = '/(\d+)-(\d{4})-(\d{2})-(\d{2})/m'; + $replace = '$2$3$4_$1'; + xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace); + + $path = '*/*.log'; + $pattern = '/(\d+)\/(\d+)(_)*.*\.log/m'; + $replace = '$1/{modifiedtime}_$2.log'; + xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace, true); + + // Dataflows savepoint reached. + upgrade_plugin_savepoint(true, 2023110901, 'tool', 'dataflows'); + } + return true; } + +/** + * Log file helper function + * + * @param string $path + * @param string $pattern + * @param string $replace + * @param bool $modifiedtimeprefix whether or not to add a datetime prefix to the new log file + * @return bool result + */ +function xmldb_tool_dataflows_logfile_rename_helper(string $path, string $pattern, string $replace, $modifiedtimeprefix = false) { + global $CFG; + + $plugindatadir = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows'; + $files = glob($plugindatadir . DIRECTORY_SEPARATOR . $path); + foreach ($files as $file) { + $strreplace = $replace; + if ($modifiedtimeprefix) { + $newprefix = date('Ymd_His000', filemtime($file)); + $strreplace = str_replace('{modifiedtime}', $newprefix, $replace); + } + $newlocation = preg_replace($pattern, $strreplace, $file); + if ($newlocation) { + rename($file, $newlocation); + } + } +} diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index a39d941a..ba22bc0f 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -47,8 +47,8 @@ $string['gpg_key_dir_desc'] = 'Path to keyring directory'; $string['log_handlers'] = 'Log handlers'; $string['log_handlers_desc'] = 'Additional log handlers to output dataflow logs to more destinations. The handler for mtrace is always active and cannot be disabled. Applying the settings at the dataflow level will override settings applied at the site admin level.'; -$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/{id}-Y-m-d.log'; -$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/{id}.log'; +$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/Ymd_{dataflowid}.log'; +$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/Ymd_his_{runid}.log'; $string['log_handler_browser_console'] = 'Browser Console'; $string['permitted_dirs'] = 'Permitted directories'; $string['permitted_dirs_desc'] = "List directories here to allow them to be read from/written to by dataflow steps. From f1129d41046483958014d5b8a03b2cb909a40170 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Fri, 22 Dec 2023 14:46:13 +1100 Subject: [PATCH 16/16] build: bump version --- version.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.php b/version.php index aeecfecd..af03ce66 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023101600; -$plugin->release = 2023101600; +$plugin->version = 2023122200; +$plugin->release = 2023122200; $plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0). // TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later. $plugin->component = 'tool_dataflows';