diff --git a/classes/DataWarehouse/Query/Cloud/JobDataset.php b/classes/DataWarehouse/Query/Cloud/JobDataset.php index b31174aad4..64fc590c1a 100644 --- a/classes/DataWarehouse/Query/Cloud/JobDataset.php +++ b/classes/DataWarehouse/Query/Cloud/JobDataset.php @@ -48,6 +48,7 @@ public function __construct( // This table is defined in the configuration file, but used in the section below. $factTable = $tables['i']; + $sessionTable = $tables['sr']; if (isset($parameters['primary_key'])) { $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'instance_id'), "=", $parameters['primary_key'])); @@ -57,33 +58,21 @@ public function __construct( } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) { date_default_timezone_set('UTC'); $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']); - $startDateTs = mktime( - 0, - 0, - 0, - $startDate['month'], - $startDate['day'], - $startDate['year'] - ); + $startDateTs = mktime(0, 0, 0, $startDate['month'], $startDate['day'], $startDate['year']); + if ($startDateTs === false) { throw new Exception('invalid "start_date" query parameter'); } $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']); - $endDateTs = mktime( - 23, - 59, - 59, - $endDate['month'], - $endDate['day'], - $endDate['year'] - ); + $endDateTs = mktime(23, 59, 59, $endDate['month'], $endDate['day'], $endDate['year']); + if ($startDateTs === false) { throw new Exception('invalid "end_date" query parameter'); } - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs)); - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'end_time_ts'), ">=", $startDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'start_time_ts'), "<=", $endDateTs)); } else { throw new Exception('invalid query parameters'); } diff --git a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php index 13704e4dc0..200220dde8 100644 --- a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php +++ b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php @@ -26,7 +26,7 @@ public function execute() if ($mysql_helper->tableExists('modw_cloud.event')) { Utilities::runEtlPipeline( - ['cloud-migration-9_0_0-9_5_0'], + ['cloud-migration-9-0-0_9-5-0'], $this->logger, [ 'last-modified-start-date' => '2017-01-01 00:00:00' @@ -59,7 +59,7 @@ public function execute() } Utilities::runEtlPipeline( - ['cloud-resource-specs-migration-9_0_0-9_5_0'], + ['cloud-resource-specs-migration-9-0-0_9-5-0'], $this->logger, [ 'last-modified-start-date' => '2017-01-01 00:00:00' diff --git a/configuration/etl/etl.d/jobs_cloud_common.json b/configuration/etl/etl.d/jobs_cloud_common.json index 34464c8d0d..8fd91318ac 100644 --- a/configuration/etl/etl.d/jobs_cloud_common.json +++ b/configuration/etl/etl.d/jobs_cloud_common.json @@ -50,6 +50,7 @@ "cloud_common/account.json", "cloud_common/event.json", "cloud_common/asset.json", + "cloud_common/host.json", "cloud_common/instance_data.json", "cloud_common/event_asset.json", "cloud_common/raw_resource_specs.json", @@ -111,6 +112,21 @@ { "#": "Note that any actions run after this cannot truncate the tables set here", + "name": "CloudHostUnknownInitializer", + "description": "Initialize value for unknown host", + "class": "StructuredFileIngestor", + "definition_file": "cloud_common/host.json", + "endpoints": { + "source": { + "type": "jsonfile", + "name": "Cloud record types", + "path": "cloud_common/host.json" + } + } + }, + { + "#": "Note that any actions run after this cannot truncate the tables set here", + "name": "CloudAccountUnknownInitializer", "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource", "class": "StructuredFileIngestor", diff --git a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json index 536da32712..c380f913f2 100644 --- a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json +++ b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json @@ -6,7 +6,7 @@ "class": "ExecuteSql", "options_class": "MaintenanceOptions" }, - "cloud-migration-9_0_0-9_5_0": { + "cloud-migration-9-0-0_9-5-0": { "namespace": "ETL\\Ingestor", "options_class": "IngestorOptions", "endpoints": { @@ -24,7 +24,7 @@ } } }, - "cloud-resource-specs-migration-9_0_0-9_5_0": { + "cloud-resource-specs-migration-9-0-0_9-5-0": { "namespace": "ETL\\Ingestor", "options_class": "IngestorOptions", "endpoints": { @@ -158,7 +158,7 @@ } } ], - "cloud-migration-9_0_0-9_5_0": [ + "cloud-migration-9-0-0_9-5-0": [ { "name": "UpdateEventTypes", "description": "Update event type on event table", @@ -169,6 +169,10 @@ { "delimiter": ";", "sql_file": "migrations/9.0.0-9.5.0/modw_cloud/update_event_types.sql" + }, + { + "delimiter": ";", + "sql_file":"migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql" } ] }, @@ -222,7 +226,7 @@ } } ], - "cloud-resource-specs-migration-9_0_0-9_5_0": [ + "cloud-resource-specs-migration-9-0-0_9-5-0": [ { "name": "CloudResourceSpecsReconstructor", "class": "CloudResourceSpecsStateTransformIngestor", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/host.json b/configuration/etl/etl_action_defs.d/cloud_common/host.json new file mode 100644 index 0000000000..664bdd3645 --- /dev/null +++ b/configuration/etl/etl_action_defs.d/cloud_common/host.json @@ -0,0 +1,8 @@ +{ + "#": "This action loads data from a structured file and only needs the table definition", + "table_definition": [ + { + "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition" + } + ] +} diff --git a/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json b/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json index a484d106c8..736d1c28d0 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json @@ -14,9 +14,6 @@ "#": "the first part of the key and not -1. We will continue to use -1 for global unknowns.", "table_definition": [ - { - "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition" - }, { "$ref": "${table_definition_dir}/cloud_common/image.json#/table_definition" } @@ -52,11 +49,6 @@ }, "destination_record_map": { - "host": { - "resource_id": "resource_id", - "host_id": "unknown_id", - "hostname": "unknown_key" - }, "image": { "resource_id": "resource_id", "image_id": "unknown_id", diff --git a/configuration/etl/etl_data.d/cloud_common/host.json b/configuration/etl/etl_data.d/cloud_common/host.json new file mode 100644 index 0000000000..f7ef0f00f5 --- /dev/null +++ b/configuration/etl/etl_data.d/cloud_common/host.json @@ -0,0 +1,4 @@ +[ + ["resource_id", "host_id", "hostname"], + [-1, 1, "Unknown"] +] diff --git a/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql b/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql new file mode 100644 index 0000000000..8ff283e049 --- /dev/null +++ b/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql @@ -0,0 +1,25 @@ +LOCK TABLES + modw_cloud.host WRITE, + modw_cloud.host as h WRITE, + modw_cloud.event WRITE, + modw_cloud.event as ev WRITE; + +ALTER TABLE modw_cloud.host MODIFY host_id INT NOT NULL; +ALTER TABLE modw_cloud.host DROP INDEX autoincrement_key; +ALTER TABLE modw_cloud.host ADD COLUMN new_host_id INT(11) UNSIGNED NOT NULL auto_increment unique; +CREATE INDEX host_resource_idx ON modw_cloud.host (host_id, resource_id); + +UPDATE + modw_cloud.event as ev +JOIN + modw_cloud.host as h +ON + ev.resource_id = h.resource_id AND ev.host_id = h.host_id +SET + ev.host_id = h.new_host_id; + +ALTER TABLE modw_cloud.host DROP COLUMN host_id; +ALTER TABLE modw_cloud.host CHANGE new_host_id host_id INT(11) UNSIGNED; +DROP INDEX host_resource_idx ON modw_cloud.host; + +UNLOCK TABLES; diff --git a/configuration/etl/etl_tables.d/cloud_common/host.json b/configuration/etl/etl_tables.d/cloud_common/host.json index e011a2843c..5696df6abf 100644 --- a/configuration/etl/etl_tables.d/cloud_common/host.json +++ b/configuration/etl/etl_tables.d/cloud_common/host.json @@ -40,15 +40,8 @@ "is_unique": true }, { - "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a", - "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column", - "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is", - "#": "useful when you want to put data into ordered groups.", - "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)", - "#": "and https://www.ryadel.com/en/mysql-two-columns-primary-key-with-auto-increment/", "name": "autoincrement_key", "columns": [ - "resource_id", "host_id" ], "is_unique": true