From f6d0a713f1e49bb18b7b651eba798cbe05666736 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Mon, 19 Apr 2021 16:37:16 -0400 Subject: [PATCH 1/3] updates for 9.5 release --- .../DataWarehouse/Query/Cloud/JobDataset.php | 41 +++++++++++-------- .../Version900To950/DatabaseMigration.php | 4 +- .../etl/etl.d/jobs_cloud_common.json | 16 ++++++++ .../etl.d/xdmod-migration-9_0_0-9_5_0.json | 12 ++++-- .../etl_action_defs.d/cloud_common/host.json | 8 ++++ .../initialize_unknown_dimension_values.json | 8 ---- .../etl/etl_data.d/cloud_common/host.json | 4 ++ .../modw_cloud/update_hosts_table_index.sql | 25 +++++++++++ .../etl/etl_tables.d/cloud_common/host.json | 7 ---- 9 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 configuration/etl/etl_action_defs.d/cloud_common/host.json create mode 100644 configuration/etl/etl_data.d/cloud_common/host.json create mode 100644 configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql diff --git a/classes/DataWarehouse/Query/Cloud/JobDataset.php b/classes/DataWarehouse/Query/Cloud/JobDataset.php index b31174aad4..959863d4b6 100644 --- a/classes/DataWarehouse/Query/Cloud/JobDataset.php +++ b/classes/DataWarehouse/Query/Cloud/JobDataset.php @@ -44,10 +44,27 @@ public function __construct( '=', new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) )); + + /*if(is_array($join[0])) { + foreach($join as $j) { + $this->addWhereCondition(new WhereCondition( + new TableField($table, $j['primaryKey']), + '=', + new TableField($tables[$j['foreignTableAlias']], $j['foreignKey']) + )); + } + } else { + $this->addWhereCondition(new WhereCondition( + new TableField($table, $join['primaryKey']), + '=', + new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) + )); + }*/ } // This table is defined in the configuration file, but used in the section below. $factTable = $tables['i']; + $sessionTable = $tables['sr']; if (isset($parameters['primary_key'])) { $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'instance_id'), "=", $parameters['primary_key'])); @@ -57,33 +74,21 @@ public function __construct( } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) { date_default_timezone_set('UTC'); $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']); - $startDateTs = mktime( - 0, - 0, - 0, - $startDate['month'], - $startDate['day'], - $startDate['year'] - ); + $startDateTs = mktime(0, 0, 0, $startDate['month'], $startDate['day'], $startDate['year']); + if ($startDateTs === false) { throw new Exception('invalid "start_date" query parameter'); } $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']); - $endDateTs = mktime( - 23, - 59, - 59, - $endDate['month'], - $endDate['day'], - $endDate['year'] - ); + $endDateTs = mktime(23, 59, 59, $endDate['month'], $endDate['day'], $endDate['year']); + if ($startDateTs === false) { throw new Exception('invalid "end_date" query parameter'); } - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs)); - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'end_time_ts'), ">=", $startDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'start_time_ts'), "<=", $endDateTs)); } else { throw new Exception('invalid query parameters'); } diff --git a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php index 13704e4dc0..200220dde8 100644 --- a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php +++ b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php @@ -26,7 +26,7 @@ public function execute() if ($mysql_helper->tableExists('modw_cloud.event')) { Utilities::runEtlPipeline( - ['cloud-migration-9_0_0-9_5_0'], + ['cloud-migration-9-0-0_9-5-0'], $this->logger, [ 'last-modified-start-date' => '2017-01-01 00:00:00' @@ -59,7 +59,7 @@ public function execute() } Utilities::runEtlPipeline( - ['cloud-resource-specs-migration-9_0_0-9_5_0'], + ['cloud-resource-specs-migration-9-0-0_9-5-0'], $this->logger, [ 'last-modified-start-date' => '2017-01-01 00:00:00' diff --git a/configuration/etl/etl.d/jobs_cloud_common.json b/configuration/etl/etl.d/jobs_cloud_common.json index 34464c8d0d..8fd91318ac 100644 --- a/configuration/etl/etl.d/jobs_cloud_common.json +++ b/configuration/etl/etl.d/jobs_cloud_common.json @@ -50,6 +50,7 @@ "cloud_common/account.json", "cloud_common/event.json", "cloud_common/asset.json", + "cloud_common/host.json", "cloud_common/instance_data.json", "cloud_common/event_asset.json", "cloud_common/raw_resource_specs.json", @@ -111,6 +112,21 @@ { "#": "Note that any actions run after this cannot truncate the tables set here", + "name": "CloudHostUnknownInitializer", + "description": "Initialize value for unknown host", + "class": "StructuredFileIngestor", + "definition_file": "cloud_common/host.json", + "endpoints": { + "source": { + "type": "jsonfile", + "name": "Cloud record types", + "path": "cloud_common/host.json" + } + } + }, + { + "#": "Note that any actions run after this cannot truncate the tables set here", + "name": "CloudAccountUnknownInitializer", "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource", "class": "StructuredFileIngestor", diff --git a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json index 3cb6007c66..4767ffd0e1 100644 --- a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json +++ b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json @@ -6,7 +6,7 @@ "class": "ExecuteSql", "options_class": "MaintenanceOptions" }, - "cloud-migration-9_0_0-9_5_0": { + "cloud-migration-9-0-0_9-5-0": { "namespace": "ETL\\Ingestor", "options_class": "IngestorOptions", "endpoints": { @@ -24,7 +24,7 @@ } } }, - "cloud-resource-specs-migration-9_0_0-9_5_0": { + "cloud-resource-specs-migration-9-0-0_9-5-0": { "namespace": "ETL\\Ingestor", "options_class": "IngestorOptions", "endpoints": { @@ -159,7 +159,7 @@ } } ], - "cloud-migration-9_0_0-9_5_0": [ + "cloud-migration-9-0-0_9-5-0": [ { "name": "UpdateEventTypes", "description": "Update event type on event table", @@ -170,6 +170,10 @@ { "delimiter": ";", "sql_file": "migrations/9.0.0-9.5.0/modw_cloud/update_event_types.sql" + }, + { + "delimiter": ";", + "sql_file":"migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql" } ] }, @@ -223,7 +227,7 @@ } } ], - "cloud-resource-specs-migration-9_0_0-9_5_0": [ + "cloud-resource-specs-migration-9-0-0_9-5-0": [ { "name": "CloudResourceSpecsReconstructor", "class": "CloudResourceSpecsStateTransformIngestor", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/host.json b/configuration/etl/etl_action_defs.d/cloud_common/host.json new file mode 100644 index 0000000000..664bdd3645 --- /dev/null +++ b/configuration/etl/etl_action_defs.d/cloud_common/host.json @@ -0,0 +1,8 @@ +{ + "#": "This action loads data from a structured file and only needs the table definition", + "table_definition": [ + { + "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition" + } + ] +} diff --git a/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json b/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json index a484d106c8..736d1c28d0 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json @@ -14,9 +14,6 @@ "#": "the first part of the key and not -1. We will continue to use -1 for global unknowns.", "table_definition": [ - { - "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition" - }, { "$ref": "${table_definition_dir}/cloud_common/image.json#/table_definition" } @@ -52,11 +49,6 @@ }, "destination_record_map": { - "host": { - "resource_id": "resource_id", - "host_id": "unknown_id", - "hostname": "unknown_key" - }, "image": { "resource_id": "resource_id", "image_id": "unknown_id", diff --git a/configuration/etl/etl_data.d/cloud_common/host.json b/configuration/etl/etl_data.d/cloud_common/host.json new file mode 100644 index 0000000000..f7ef0f00f5 --- /dev/null +++ b/configuration/etl/etl_data.d/cloud_common/host.json @@ -0,0 +1,4 @@ +[ + ["resource_id", "host_id", "hostname"], + [-1, 1, "Unknown"] +] diff --git a/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql b/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql new file mode 100644 index 0000000000..8ff283e049 --- /dev/null +++ b/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql @@ -0,0 +1,25 @@ +LOCK TABLES + modw_cloud.host WRITE, + modw_cloud.host as h WRITE, + modw_cloud.event WRITE, + modw_cloud.event as ev WRITE; + +ALTER TABLE modw_cloud.host MODIFY host_id INT NOT NULL; +ALTER TABLE modw_cloud.host DROP INDEX autoincrement_key; +ALTER TABLE modw_cloud.host ADD COLUMN new_host_id INT(11) UNSIGNED NOT NULL auto_increment unique; +CREATE INDEX host_resource_idx ON modw_cloud.host (host_id, resource_id); + +UPDATE + modw_cloud.event as ev +JOIN + modw_cloud.host as h +ON + ev.resource_id = h.resource_id AND ev.host_id = h.host_id +SET + ev.host_id = h.new_host_id; + +ALTER TABLE modw_cloud.host DROP COLUMN host_id; +ALTER TABLE modw_cloud.host CHANGE new_host_id host_id INT(11) UNSIGNED; +DROP INDEX host_resource_idx ON modw_cloud.host; + +UNLOCK TABLES; diff --git a/configuration/etl/etl_tables.d/cloud_common/host.json b/configuration/etl/etl_tables.d/cloud_common/host.json index e011a2843c..5696df6abf 100644 --- a/configuration/etl/etl_tables.d/cloud_common/host.json +++ b/configuration/etl/etl_tables.d/cloud_common/host.json @@ -40,15 +40,8 @@ "is_unique": true }, { - "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a", - "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column", - "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is", - "#": "useful when you want to put data into ordered groups.", - "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)", - "#": "and https://www.ryadel.com/en/mysql-two-columns-primary-key-with-auto-increment/", "name": "autoincrement_key", "columns": [ - "resource_id", "host_id" ], "is_unique": true From 145566069d1447cbb7d34a92ad29edbdd2777b6e Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Mon, 19 Apr 2021 16:38:23 -0400 Subject: [PATCH 2/3] updates for 9.5 release --- .../DataWarehouse/Query/Cloud/JobDataset.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/classes/DataWarehouse/Query/Cloud/JobDataset.php b/classes/DataWarehouse/Query/Cloud/JobDataset.php index 959863d4b6..5b8dc3cbaf 100644 --- a/classes/DataWarehouse/Query/Cloud/JobDataset.php +++ b/classes/DataWarehouse/Query/Cloud/JobDataset.php @@ -45,23 +45,6 @@ public function __construct( new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) )); - /*if(is_array($join[0])) { - foreach($join as $j) { - $this->addWhereCondition(new WhereCondition( - new TableField($table, $j['primaryKey']), - '=', - new TableField($tables[$j['foreignTableAlias']], $j['foreignKey']) - )); - } - } else { - $this->addWhereCondition(new WhereCondition( - new TableField($table, $join['primaryKey']), - '=', - new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) - )); - }*/ - } - // This table is defined in the configuration file, but used in the section below. $factTable = $tables['i']; $sessionTable = $tables['sr']; From 4e16085fc40cac11d7bc6e519a6185ce771bd973 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Tue, 20 Apr 2021 08:47:58 -0400 Subject: [PATCH 3/3] adding missing brace --- classes/DataWarehouse/Query/Cloud/JobDataset.php | 1 + 1 file changed, 1 insertion(+) diff --git a/classes/DataWarehouse/Query/Cloud/JobDataset.php b/classes/DataWarehouse/Query/Cloud/JobDataset.php index 5b8dc3cbaf..64fc590c1a 100644 --- a/classes/DataWarehouse/Query/Cloud/JobDataset.php +++ b/classes/DataWarehouse/Query/Cloud/JobDataset.php @@ -44,6 +44,7 @@ public function __construct( '=', new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) )); + } // This table is defined in the configuration file, but used in the section below. $factTable = $tables['i'];