Merge pull request #934 from besawn/csm_bds_python

Update the CSM BDS python use cases to support python3 on RHEL8

csm_big_data/logstash/plugins/csm_event_correlator/logstash-filter-csm_event_correlator.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-csm-event-correlator'
   s.version         = "0.1.0"
-  s.licenses = ["Eclipse Public License v1.0"]
+  s.licenses = ["EPL-1.0"]
   s.summary = "A filter for generating RAS events in response to log patterns."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["John Dunham"]

csm_big_data/python/cast_helper.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    cast_helper.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,15 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
+
 '''
 .. module:: cast_helper
     :platform: Linux
@@ -30,6 +39,7 @@
 from elasticsearch.serializer import JSONSerializer
 from elasticsearch import exceptions
 from datetime import datetime
+from functools import reduce
    
 TARGET_ENV='CAST_ELASTIC'
 
@@ -307,15 +317,15 @@ def search_job( es, allocation_id=0, primary_job_id=0, secondary_job_id=0, size=
 
     query={ "bool" : { "should" : [] } }
 
-    if allocation_id > 0 : 
+    if int(allocation_id) > 0 : 
         query={ 
             "bool" : { 
                 "should" : [ 
                     { "match" : { "data.allocation_id" : allocation_id } } 
                 ] 
             } 
         }
-    elif primary_job_id > 0 and secondary_job_id >= 0:
+    elif int(primary_job_id) > 0 and int(secondary_job_id) >= 0:
         query={ 
             "bool" : { 
                 "should" : [ 

csm_big_data/python/findJobKeys.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findJobKeys.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,14 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
 
 import argparse
 import sys

csm_big_data/python/findJobMetrics.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findJobMetrics.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,15 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
+
 import argparse
 import sys
 import os

csm_big_data/python/findJobTimeRange.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findJobTimeRange.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,14 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
 
 import argparse
 import sys

csm_big_data/python/findJobsInRange.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findJobsInRange.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,15 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
+
 import argparse
 import sys
 import os

csm_big_data/python/findJobsRunning.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findJobsRunning.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,15 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
+
 import argparse
 import sys
 import os

csm_big_data/python/findUserJobs.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findUserJobs.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,6 +15,14 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
 
 import argparse
 import sys
@@ -147,7 +155,7 @@ def main(args):
         print("{0:>{1}} : {2}".format("node", max_width, "common count"))
 
         node_count=int(args.commonnodes)
-        for key,value in sorted( node_collisions.iteritems(), key=lambda (k,v): (v,k), reverse=False):
+        for key,value in sorted( node_collisions.iteritems(), key=lambda k_v: (k_v[1],k_v[0]), reverse=False):
             if int(value) > node_count:
                 collision_found = True
                 print("{0:>{1}} : {2}".format(key, max_width, value))

csm_big_data/python/findWeightedErrors.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python2
+#!/bin/sh
 # encoding: utf-8
 #================================================================================
 #
 #    findWeightedErrors.py
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -15,10 +15,17 @@
 #
 #================================================================================
 
+# The beginning of this script is both valid shell and valid python,
+# such that the script starts with the shell and is reexecuted with
+# the right python.
+#
+# The intent is to run as python3 on RHEL8 and python2 on RHEL7
+#
+'''which' python3 > /dev/null 2>&1 && exec python3 "$0" "$@" || exec python2 "$0" "$@"
+'''
 
 import argparse
 import json
-import sets
 import sys
 import os
 
@@ -235,8 +242,8 @@ def main(args):
 
     print(" ")
     # Print the results.
-    for category, response in sorted(results.iteritems(), 
-            key=lambda (k, v): cast.deep_get( v, "hits", "max_score" ), reverse=True ):
+    for category, response in sorted(results.iteritems(),
+            key=lambda k_v1: cast.deep_get( k_v1[1], "hits", "max_score" ), reverse=True ):
 
         # Get aggregations. 
         aggregations = response.get("aggregations", [])
@@ -249,7 +256,7 @@ def main(args):
         if aggregations is not None:
             # Sort aggregations by document count.
             for (aggregation,value) in sorted(aggregations.iteritems(), 
-                    key=lambda (k, v): v.get("doc_count"), reverse=True):
+                    key=lambda k_v2: k_v2[1].get("doc_count"), reverse=True):
                 print("  \"{0}\" : {1}".format( aggregation, value.get("doc_count") ) ) 
 
         if args.verbose:

csm_big_data/setupRPM.cmake

@@ -1,8 +1,8 @@
 #================================================================================
 #
-#    csm_big_data/castBDS.cmake
+#    csm_big_data/setupRPM.cmake
 #
-#    © Copyright IBM Corporation 2015-2018. All Rights Reserved
+#    © Copyright IBM Corporation 2015-2020. All Rights Reserved
 #
 #    This program is licensed under the terms of the Eclipse Public License
 #    v1.0 as published by the Eclipse Foundation and available at
@@ -20,6 +20,8 @@ set( CPACK_RPM_csm-bds_PRE_UNINSTALL_SCRIPT_FILE
     "${CMAKE_CURRENT_SOURCE_DIR}/csm_big_data/rpmscripts/cast-bds.pre.uninstall" )
 if(${EXPLICIT_PYTHON_VERSION})
    set(CPACK_RPM_csm-bds_PACKAGE_REQUIRES "python2-psycopg2 >= 2.5.1")
+   set(CPACK_RPM_csm-bds_PACKAGE_REQUIRES "python3-dateutil >= 2.6")
+   set(CPACK_RPM_csm-bds_PACKAGE_REQUIRES "python3-elasticsearch >= 7.0")
 else()
    set(CPACK_RPM_csm-bds_PACKAGE_REQUIRES "python-psycopg2 >= 2.5.1")
 endif()