SCTASK0066541 Merge pull request #124 from veorlo/master

SCTASK0066541 Version 1.6.1

Makefile

@@ -1,5 +1,5 @@
 NAME=simp
-VERSION=1.6.0
+VERSION=1.6.1
 
 .PHONY: dist
 

lib/GRNOC/Simp/Comp.pm

@@ -15,7 +15,7 @@ use GRNOC::Config;
 use GRNOC::Log;
 use GRNOC::Simp::Comp::Worker;
 
-our $VERSION = '1.6.0';
+our $VERSION = '1.6.1';
 
 ### REQUIRED ATTRIBUTES ###
 
@@ -277,12 +277,15 @@ sub _make_composites {
         $composite{'scans'} = [];
         if (exists $variables->{'scan'} && ref $variables->{'scan'} eq 'ARRAY') {
 
-            for my $scan (@{$variables->{'scan'}}) {
+            for (my $i = 0; $i <= $#{$variables->{'scan'}}; $i++) {
+
+                my $scan = $variables->{'scan'}[$i];
 
                 my $scan_params = {
                     'oid'    => $scan->{'oid'},
                     'suffix' => $scan->{oid_suffix},
-                    'value'  => $scan->{poll_value}
+                    'value'  => $scan->{poll_value},
+                    'index'  => $i # Index used to preserve ordering for async responses
                 };
 
                 $self->_map_oid($scan_params, 'scan');

lib/GRNOC/Simp/Comp/Worker.pm

@@ -431,11 +431,8 @@ sub _gather_data {
     $self->logger->debug('['.$composite->{name}."] Gathering data from Redis...");
 
     # Gather data for scans
-    # Keep track of what index this scan is so we can store it correctly,
-    # these might return in a different order because async
-    my $scan_index = 0;
     for my $scan (@{$composite->{'scans'}}) {   
-        $self->_request_data($request, $scan->{value}, $scan, $scan_index++, $cv);
+        $self->_request_data($request, $scan->{value}, $scan, $cv);
     }
 
     # Gather data for data elements
@@ -447,7 +444,7 @@ sub _gather_data {
         # Skip data elements where their source is a constant
         next if ($attr->{source_type} eq 'constant');
 
-        $self->_request_data($request, $name, $attr, undef, $cv);
+        $self->_request_data($request, $name, $attr, $cv);
     }
 
     # Signal that we have finished gathering all of the data
@@ -461,7 +458,7 @@ sub _gather_data {
     OID data is cached in $request->{raw} by the callback.
 =cut
 sub _request_data {
-    my ($self, $request, $name, $attr, $scan_index, $cv) = @_;
+    my ($self, $request, $name, $attr, $cv) = @_;
 
     # Get the base OID subtree to request.
     # It is tempting to request just the OIDs you know you want,
@@ -470,7 +467,6 @@ sub _request_data {
     # more time and CPU. That's why we request the subtree.
     my $oid = $attr->{base_oid};
 
-
     # Add to the AnyEvent condition variable to make the request async
     # We end it within the callback after data caching completes
     $cv->begin;
@@ -486,7 +482,7 @@ sub _request_data {
             oidmatch       => [$oid],
             async_callback => sub {
                 my $data = shift;
-                $self->_cache_data($request, $name, $attr, $scan_index, $data->{results});
+                $self->_cache_data($request, $name, $attr, $data->{results});
                 $cv->end;
             }
         );
@@ -498,7 +494,7 @@ sub _request_data {
             oidmatch       => [$oid],
             async_callback => sub {
                 my $data = shift;
-                $self->_cache_data($request, $name, $attr, $scan_index, $data->{results});
+                $self->_cache_data($request, $name, $attr, $data->{results});
                 $cv->end;
             }
         );
@@ -513,7 +509,7 @@ sub _request_data {
     OIDs returned from Redis are checked to see if they match what was requested.
 =cut
 sub _cache_data {
-    my ($self, $request, $name, $attr, $scan_index, $data) = @_;
+    my ($self, $request, $name, $attr, $data) = @_;
 
     my $composite_name = $request->{composite}{name};
 
@@ -531,18 +527,15 @@ sub _cache_data {
     for my $host (@{$request->{hosts}}) {
         for my $port (keys(%$data)) {
 
-            # Raw scan data for a host needs to be wrapped in an array per individual scan
-            my @scan_arr;
-
             # Skip the host when there's no data for it
             if (!exists($data->{$port}{$host}) || !defined($data->{$port}{$host})) {
                 $self->logger->error("[$composite_name] $host has no data to cache for \"$name\"");
                 next;
             }
 
-            # For scans, we want an ordered array of arrays where the outer array indicates which scan we want
-            my @output;
- 
+            # Raw scan data for a host needs to be wrapped in an array per individual scan
+            my @scan_arr;
+
             for my $oid (keys %{$data->{$port}{$host}}) {
 
                 my $value = $data->{$port}{$host}{$oid}{value};
@@ -571,14 +564,22 @@ sub _cache_data {
 
             # Push the array of raw data for the scan
             if ($type eq 'scan') {
+
+                # Initialize the port hash for hosts
                 unless (exists($request->{raw}{$type}{$port})) {
                     $request->{raw}{$type}{$port} = {};
                 }
+                # Initialize the array of scans for the host and port
                 unless (exists $request->{raw}{$type}{$port}{$host}) {
                     $request->{raw}{$type}{$port}{$host} = [];
                 }
 
-                $request->{raw}{$type}{$port}{$host}->[$scan_index] = \@scan_arr;
+                # Get the scan's index to preserve scan ordering
+                # Async responses from multiple simp-data workers can cause unordered scan data caching
+                my $scan_index = $attr->{index};
+
+                # Assign the scan data to the appropriate index
+                $request->{raw}{$type}{$port}{$host}[$scan_index] = \@scan_arr;
             }
         }
     }

lib/GRNOC/Simp/Data.pm

@@ -13,7 +13,7 @@ use GRNOC::Config;
 use GRNOC::Log;
 use GRNOC::Simp::Data::Worker;
 
-our $VERSION = '1.6.0';
+our $VERSION = '1.6.1';
 
 =head2 public attributes
 =over 12

lib/GRNOC/Simp/Poller.pm

@@ -15,7 +15,7 @@ use GRNOC::Config;
 use GRNOC::Log;
 use GRNOC::Simp::Poller::Worker;
 
-our $VERSION = '1.6.0';
+our $VERSION = '1.6.1';
 
 ### Required Attributes ###
 =head2 public attributes

lib/GRNOC/Simp/TSDS.pm

@@ -3,7 +3,7 @@ package GRNOC::Simp::TSDS;
 use strict;
 use warnings;
 
-our $VERSION = '1.6.0';
+our $VERSION = '1.6.1';
 
 =head1 NAME
 

simp-comp.spec

@@ -1,6 +1,6 @@
 Summary: A system for fetching data from simp and compiling the data into a composite
 Name: simp-comp
-Version: 1.6.0
+Version: 1.6.1
 Release: 1%{dist}
 License: GRNOC
 Group: GRNOC

simp-data.spec

@@ -1,6 +1,6 @@
 Summary: A small system for fetching SNMP data from redis and returning it via RabbitMQ
 Name: simp-data
-Version: 1.6.0
+Version: 1.6.1
 Release: 1%{dist}
 License: GRNOC
 Group: GRNOC

simp-monitor.spec

@@ -1,5 +1,5 @@
 Name: simp-monitor
-Version: 1.6.0
+Version: 1.6.1
 Release: 1%{?dist}
 Summary: A functionality to monitor SIMP
 License: GRNOC

simp-poller.spec

@@ -1,6 +1,6 @@
 Summary: A small system for gathering large amounts of SNMP data and pushing them into redis
 Name: simp-poller
-Version: 1.6.0
+Version: 1.6.1
 Release: 1%{dist}
 License: GRNOC
 Group: GRNOC

simp-tsds.spec

@@ -1,6 +1,6 @@
 Summary: SIMP TSDS Collector
 Name: simp-tsds
-Version: 1.6.0
+Version: 1.6.1
 Release: 1%{dist}
 License: APL 2.0
 Group: Network