Merge pull request #1527 from eiffel777/update-raw-stats-correct-time…

…-field Updates for cloud Data Export and migration pipelines.
ubccr · Apr 21, 2021 · 0f8d1b7 · 0f8d1b7
2 parents f589a9e + 4e16085
commit 0f8d1b7
Show file tree

Hide file tree

Showing 9 changed files with 70 additions and 39 deletions.
diff --git a/classes/DataWarehouse/Query/Cloud/JobDataset.php b/classes/DataWarehouse/Query/Cloud/JobDataset.php
@@ -48,6 +48,7 @@ public function __construct(
 
         // This table is defined in the configuration file, but used in the section below.
         $factTable = $tables['i'];
+        $sessionTable = $tables['sr'];
 
         if (isset($parameters['primary_key'])) {
             $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'instance_id'), "=", $parameters['primary_key']));
@@ -57,33 +58,21 @@ public function __construct(
         } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) {
             date_default_timezone_set('UTC');
             $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']);
-            $startDateTs = mktime(
-                0,
-                0,
-                0,
-                $startDate['month'],
-                $startDate['day'],
-                $startDate['year']
-            );
+            $startDateTs = mktime(0, 0, 0, $startDate['month'], $startDate['day'], $startDate['year']);
+
             if ($startDateTs === false) {
                 throw new Exception('invalid "start_date" query parameter');
             }
 
             $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']);
-            $endDateTs = mktime(
-                23,
-                59,
-                59,
-                $endDate['month'],
-                $endDate['day'],
-                $endDate['year']
-            );
+            $endDateTs = mktime(23, 59, 59, $endDate['month'], $endDate['day'], $endDate['year']);
+
             if ($startDateTs === false) {
                 throw new Exception('invalid "end_date" query parameter');
             }
 
-            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs));
-            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs));
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'end_time_ts'), ">=", $startDateTs));
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($sessionTable, 'start_time_ts'), "<=", $endDateTs));
         } else {
             throw new Exception('invalid query parameters');
         }

diff --git a/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php b/classes/OpenXdmod/Migration/Version900To950/DatabaseMigration.php
@@ -26,7 +26,7 @@ public function execute()
         if ($mysql_helper->tableExists('modw_cloud.event')) {
 
             Utilities::runEtlPipeline(
-                ['cloud-migration-9_0_0-9_5_0'],
+                ['cloud-migration-9-0-0_9-5-0'],
                 $this->logger,
                 [
                     'last-modified-start-date' => '2017-01-01 00:00:00'
@@ -59,7 +59,7 @@ public function execute()
             }
 
             Utilities::runEtlPipeline(
-                ['cloud-resource-specs-migration-9_0_0-9_5_0'],
+                ['cloud-resource-specs-migration-9-0-0_9-5-0'],
                 $this->logger,
                 [
                     'last-modified-start-date' => '2017-01-01 00:00:00'

diff --git a/configuration/etl/etl.d/jobs_cloud_common.json b/configuration/etl/etl.d/jobs_cloud_common.json
@@ -50,6 +50,7 @@
                 "cloud_common/account.json",
                 "cloud_common/event.json",
                 "cloud_common/asset.json",
+                "cloud_common/host.json",
                 "cloud_common/instance_data.json",
                 "cloud_common/event_asset.json",
                 "cloud_common/raw_resource_specs.json",
@@ -111,6 +112,21 @@
         {
             "#": "Note that any actions run after this cannot truncate the tables set here",
 
+            "name": "CloudHostUnknownInitializer",
+            "description": "Initialize value for unknown host",
+            "class": "StructuredFileIngestor",
+            "definition_file": "cloud_common/host.json",
+            "endpoints": {
+                "source": {
+                    "type": "jsonfile",
+                    "name": "Cloud record types",
+                    "path": "cloud_common/host.json"
+                }
+            }
+        },
+        {
+            "#": "Note that any actions run after this cannot truncate the tables set here",
+
             "name": "CloudAccountUnknownInitializer",
             "description": "Initialize values for unknown hosts, images, etc. that are specific to a resource",
             "class": "StructuredFileIngestor",

diff --git a/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json b/configuration/etl/etl.d/xdmod-migration-9_0_0-9_5_0.json
@@ -6,7 +6,7 @@
             "class": "ExecuteSql",
             "options_class": "MaintenanceOptions"
         },
-        "cloud-migration-9_0_0-9_5_0": {
+        "cloud-migration-9-0-0_9-5-0": {
             "namespace": "ETL\\Ingestor",
             "options_class": "IngestorOptions",
             "endpoints": {
@@ -24,7 +24,7 @@
               }
             }
          },
-         "cloud-resource-specs-migration-9_0_0-9_5_0": {
+         "cloud-resource-specs-migration-9-0-0_9-5-0": {
              "namespace": "ETL\\Ingestor",
              "options_class": "IngestorOptions",
              "endpoints": {
@@ -158,7 +158,7 @@
             }
         }
     ],
-    "cloud-migration-9_0_0-9_5_0": [
+    "cloud-migration-9-0-0_9-5-0": [
         {
           "name": "UpdateEventTypes",
           "description": "Update event type on event table",
@@ -169,6 +169,10 @@
               {
                   "delimiter": ";",
                   "sql_file": "migrations/9.0.0-9.5.0/modw_cloud/update_event_types.sql"
+              },
+              {
+                  "delimiter": ";",
+                  "sql_file":"migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql"
               }
           ]
         },
@@ -222,7 +226,7 @@
             }
         }
     ],
-    "cloud-resource-specs-migration-9_0_0-9_5_0": [
+    "cloud-resource-specs-migration-9-0-0_9-5-0": [
         {
             "name": "CloudResourceSpecsReconstructor",
             "class": "CloudResourceSpecsStateTransformIngestor",

diff --git a/configuration/etl/etl_action_defs.d/cloud_common/host.json b/configuration/etl/etl_action_defs.d/cloud_common/host.json
@@ -0,0 +1,8 @@
+{
+    "#": "This action loads data from a structured file and only needs the table definition",
+    "table_definition": [
+        {
+            "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition"
+        }
+    ]
+}
diff --git a/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json b/configuration/etl/etl_action_defs.d/cloud_common/initialize_unknown_dimension_values.json
@@ -14,9 +14,6 @@
     "#": "the first part of the key and not -1. We will continue to use -1 for global unknowns.",
 
     "table_definition": [
-        {
-            "$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition"
-        },
         {
             "$ref": "${table_definition_dir}/cloud_common/image.json#/table_definition"
         }
@@ -52,11 +49,6 @@
     },
 
     "destination_record_map": {
-        "host": {
-            "resource_id": "resource_id",
-            "host_id": "unknown_id",
-            "hostname": "unknown_key"
-        },
         "image": {
             "resource_id": "resource_id",
             "image_id": "unknown_id",

diff --git a/configuration/etl/etl_data.d/cloud_common/host.json b/configuration/etl/etl_data.d/cloud_common/host.json
@@ -0,0 +1,4 @@
+[
+    ["resource_id", "host_id", "hostname"],
+    [-1,            1,         "Unknown"]
+]
diff --git a/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql b/configuration/etl/etl_sql.d/migrations/9.0.0-9.5.0/modw_cloud/update_hosts_table_index.sql
@@ -0,0 +1,25 @@
+LOCK TABLES
+  modw_cloud.host WRITE,
+  modw_cloud.host as h WRITE,
+  modw_cloud.event WRITE,
+  modw_cloud.event as ev WRITE;
+
+ALTER TABLE modw_cloud.host MODIFY host_id INT NOT NULL;
+ALTER TABLE modw_cloud.host DROP INDEX autoincrement_key;
+ALTER TABLE modw_cloud.host ADD COLUMN new_host_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+CREATE INDEX host_resource_idx ON modw_cloud.host (host_id, resource_id);
+
+UPDATE
+  modw_cloud.event as ev
+JOIN
+  modw_cloud.host as h
+ON
+  ev.resource_id = h.resource_id AND ev.host_id = h.host_id
+SET
+  ev.host_id = h.new_host_id;
+
+ALTER TABLE modw_cloud.host DROP COLUMN host_id;
+ALTER TABLE modw_cloud.host CHANGE new_host_id host_id INT(11) UNSIGNED;
+DROP INDEX host_resource_idx ON modw_cloud.host;
+
+UNLOCK TABLES;
diff --git a/configuration/etl/etl_tables.d/cloud_common/host.json b/configuration/etl/etl_tables.d/cloud_common/host.json
@@ -40,15 +40,8 @@
                 "is_unique": true
             },
             {
-                "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a",
-                "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column",
-                "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is",
-                "#": "useful when you want to put data into ordered groups.",
-                "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)",
-                "#": "and https://www.ryadel.com/en/mysql-two-columns-primary-key-with-auto-increment/",
                 "name": "autoincrement_key",
                 "columns": [
-                    "resource_id",
                     "host_id"
                 ],
                 "is_unique": true