Add support for Elasticsearch 7.x

README.md

@@ -154,7 +154,12 @@ st2 run elasticsearch.snapshots.delete host=elk repository=my_backup all_indices
 
 ### Querying Elasticsearch
 
-On successful search (*total hits > 0*) query actions exit with *return code* == 0, if no documents have been found *return code* == 1. In all other case such as execution exceptions *return code* is 99.
+| Return Code | Reason                                                      |
+|-------------|-------------------------------------------------------------|
+| `0`         | Successful search (total hits > 0 or returned hits > 0)     |
+| `1`         | No documents found (total hits == 0)                        |
+| `2`         | `hits.total` not present in response and returned hits == 0 |
+| `99`        | Other execution errors                                      |
 
 Let's look at a few examples:
 

actions/lib/utils.py

@@ -7,7 +7,7 @@
 from curator.utils import get_version, is_master_node
 
 # Elasticsearch versions supported
-version_max = (7, 0, 0)
+version_max = (8, 0, 0)
 version_min = (5, 0, 0)
 logger = logging.getLogger(__name__)
 

actions/search.py

@@ -87,7 +87,21 @@ def _pp_exit(self, data):
             kwargs = {'indent': 4}
         print(json.dumps(data, **kwargs))
 
-        if data['hits']['total'] > 0:
-            sys.exit(0)
-        else:
-            sys.exit(1)
+        # in ElasticSearch 7.0 hits.total becomes an object and may not even
+        # be present when track_total_hits is false see:
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#hits-total-now-object-search-response # noqa
+        if 'total' in data['hits']:
+            if isinstance(data['hits']['total'], int):
+                hit_value = data['hits']['total']
+            elif isinstance(data['hits']['total'], dict):
+                hit_value = data['hits']['total']['value']
+            else:
+                print('Unsupported data type for `hits.total`', file=sys.stderr)
+                sys.exit(99)
+
+            if hit_value == 0:
+                sys.exit(1)
+        elif len(data['hits']['hits']) == 0:
+            sys.exit(2)
+
+        sys.exit(0)

requirements.txt

@@ -1,2 +1,19 @@
-elasticsearch-curator==5.4.0
-easydict
+boto3==1.14.17
+botocore==1.17.17
+certifi==2020.6.20
+chardet==3.0.4
+click==6.7
+docutils==0.15.2
+easydict==1.9
+elasticsearch==7.8.0
+elasticsearch-curator==5.8.1
+idna==2.10
+jmespath==0.10.0
+python-dateutil==2.8.1
+PyYAML==3.13
+requests==2.24.0
+requests-aws4auth==1.0
+s3transfer==0.3.3
+six==1.15.0
+urllib3==1.24.3
+voluptuous==0.11.7

sensors/count_sensor.py

@@ -8,9 +8,9 @@ class ElasticsearchCountSensor(PollingSensor):
 
     def setup(self):
         self.host = self.config.get('host', None)
-        self.port = self.config.get('port', None)
-        self.http_auth = self.config.get('http_auth', None)
-        self.use_ssl = self.config.get('use_ssl', False)
+        self.port = self.config.get('port', None)
+        self.http_auth = self.config.get('http_auth', None)
+        self.use_ssl = self.config.get('use_ssl', False)
         self.query_window = self.config.get('query_window', 60)
         self.query_string = self.config.get('query_string', '{}')
         self.cooldown_multiplier = self.config.get('cooldown_multiplier', 0)
@@ -22,9 +22,9 @@ def setup(self):
         self.es = None
 
         try:
-            self.es = Elasticsearch([{'host': self.host, 'port': self.port,
-                                    'http_auth': self.http_auth,
-                                    'use_ssl': self.use_ssl}])
+            self.es = Elasticsearch([{'host': self.host, 'port': self.port,
+                                    'http_auth': self.http_auth,
+                                    'use_ssl': self.use_ssl}])
         except Exception:
             raise
 
@@ -40,7 +40,23 @@ def poll(self):
         data = self.es.search(index=self.index, body=query_payload)
 
         hits = data.get('hits', None)
-        if hits.get('total', 0) > self.count_threshold:
+        if hits is None:
+            raise RuntimeError('No hits in search response.')
+
+        total = hits.get('total', None)
+        if total is None:
+            raise RuntimeError('Total not present in search response.' +
+                               'Did you disable track_total_hits?\n' +
+                               'https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#hits-total-omitted-if-disabled')  # noqa
+
+        if isinstance(int, total):
+            hit_count = total
+        elif isinstance(dict, total):  # for Elasticsearch >= 7.0
+            hit_count = total['value']
+        else:
+            raise RuntimeError('Unsupported type of hit.total')
+
+        if hit_count > self.count_threshold:
             payload = dict()
             payload['results'] = hits
             payload['results']['query'] = query_payload