IMPALA-13124: Migrate tests that use 'unittest' package to pytest bas…

…e class

Some tests were written to use the builtin 'unittest' package.
In testing on Python 3, these tests failed with an error
like "RuntimeError: generator raised StopIteration". Since Impala
tests are standardized on pytests, this converts those locations
to use our regular pytest base classes.

This required restructing the test_redaction.py custom cluster
test to use the pytest setup and teardown methods. It also simplifies
the test cases so that each attempted startup gets its own test
rather than doing multiple startup attempts in a single test.

Testing:
 - Ran exhaustive job

Change-Id: I89e854f64e424a75827929a4f6841066024390e9
Reviewed-on: http://gerrit.cloudera.org:8080/21475
Reviewed-by: Michael Smith <[email protected]>
Reviewed-by: Riza Suminto <[email protected]>
Tested-by: Joe McDonnell <[email protected]>

tests/custom_cluster/test_redaction.py

@@ -21,7 +21,6 @@
 import os
 import pytest
 import re
-import unittest
 
 from time import sleep
 
@@ -32,9 +31,7 @@
 LOG = logging.getLogger(__name__)
 
 
-# This class needs to inherit from unittest.TestCase otherwise py.test will ignore it
-# because a __init__ method is used.
-class TestRedaction(CustomClusterTestSuite, unittest.TestCase):
+class TestRedaction(CustomClusterTestSuite):
   '''Test various redaction related functionality.
 
      Redaction is about preventing sensitive data from leaking into logs, the web ui,
@@ -46,13 +43,6 @@ class TestRedaction(CustomClusterTestSuite, unittest.TestCase):
   def get_workload(cls):
     return 'functional-query'
 
-  def __init__(self, *args, **kwargs):
-    super(TestRedaction, self).__init__(*args, **kwargs)
-
-    # Parent dir for various file output such as logging. The value is set to a new value
-    # just before each test run.
-    self.tmp_dir = None
-
   @property
   def log_dir(self):
     return os.path.join(self.tmp_dir, "logs")
@@ -71,7 +61,8 @@ def rules_file(self):
 
   def setup_method(self, method):
     # Override parent
-    pass
+    # The temporary directory gets removed in teardown_method() after each test.
+    self.tmp_dir = self.make_tmp_dir('redaction')
 
   def teardown_method(self, method):
     # Parent method would fail, nothing needs to be done.
@@ -80,7 +71,7 @@ def teardown_method(self, method):
 
   def start_cluster_using_rules(self, redaction_rules, log_level=2, vmodule=""):
     '''Start Impala with a custom log dir and redaction rules.'''
-    self.tmp_dir = self.make_tmp_dir('redaction')
+    assert self.tmp_dir
     os.chmod(self.tmp_dir, 0o777)
     LOG.info("tmp_dir is " + self.tmp_dir)
     os.mkdir(self.log_dir)
@@ -176,14 +167,11 @@ def test_bad_rules(self):
     self.assert_server_fails_to_start('{ "version": 100 }', startup_options,
         'Error parsing redaction rules; only version 1 is supported')
 
-  @pytest.mark.execute_serially
-  def test_very_verbose_logging(self):
-    '''Check that the server fails to start if logging is configured at a level that
-       could dump table data. Row logging would be enabled with "-v=3" or could be
-       enabled  with the -vmodule option. In either case the server should not start.
+  def assert_too_verbose_logging(self, start_options):
+    '''Assert that the server fails to start with the specific start_options while
+       using a basic redaction policy to redact emails. This is intended to test
+       cases where logging is configured at a level that could dump table data.
     '''
-    if self.exploration_strategy() != 'exhaustive':
-      pytest.skip('runs only in exhaustive')
     rules = r"""
         {
           "version": 1,
@@ -198,10 +186,29 @@ def test_very_verbose_logging(self):
         }"""
     error_message = "Redaction cannot be used in combination with log level 3 or " \
         "higher or the -vmodule option"
-    self.assert_server_fails_to_start(rules, {"log_level": 3}, error_message)
-    self.assert_server_fails_to_start(rules, {"vmodule": "foo"}, error_message)
-    self.assert_server_fails_to_start(
-        rules, {"log_level": 3, "vmodule": "foo"}, error_message)
+    self.assert_server_fails_to_start(rules, start_options, error_message)
+
+  @pytest.mark.execute_serially
+  def test_too_verbose_v3(self):
+    '''Check that the server fails to start when redaction is combined with -v=3.'''
+    if self.exploration_strategy() != 'exhaustive':
+      pytest.skip('runs only in exhaustive')
+    self.assert_too_verbose_logging({"log_level": 3})
+
+  @pytest.mark.execute_serially
+  def test_too_verbose_vmodule(self):
+    '''Check that the server fails to start when redaction is combined with -vmodule'''
+    if self.exploration_strategy() != 'exhaustive':
+      pytest.skip('runs only in exhaustive')
+    self.assert_too_verbose_logging({"vmodule": "foo"})
+
+  @pytest.mark.execute_serially
+  def test_too_verbose_v3_vmodule(self):
+    '''Check that the server fails to start when redaction is combined with -v=3
+       and -vmodule'''
+    if self.exploration_strategy() != 'exhaustive':
+      pytest.skip('runs only in exhaustive')
+    self.assert_too_verbose_logging({"log_level": 3, "vmodule": "foo"})
 
   @pytest.mark.execute_serially
   def test_unredacted(self):

tests/shell/test_cookie_util.py

@@ -21,7 +21,8 @@
 from __future__ import absolute_import, division, print_function
 
 import sys
-import unittest
+
+from tests.common.base_test_suite import BaseTestSuite
 
 from datetime import datetime, timedelta
 from http.client import HTTPMessage
@@ -30,7 +31,7 @@
                                get_all_matching_cookies)
 
 
-class TestCookieUtil(unittest.TestCase):
+class TestCookieUtil(BaseTestSuite):
     def test_cookie_matches_path(self):
         assert cookie_matches_path({}, '/')
         assert cookie_matches_path({}, '')

tests/shell/test_kerberos_util.py

@@ -20,12 +20,11 @@
 
 from __future__ import absolute_import, division, print_function
 
-import unittest
-
 from shell.kerberos_util import get_kerb_host_from_kerberos_host_fqdn
+from tests.common.base_test_suite import BaseTestSuite
 
 
-class TestKerberosUtil(unittest.TestCase):
+class TestKerberosUtil(BaseTestSuite):
   def test_get_kerb_host_from_kerberos_host_fqdn(self):
     assert isinstance(get_kerb_host_from_kerberos_host_fqdn("any.host:1234"), str)
     assert get_kerb_host_from_kerberos_host_fqdn("any.host:1234") == "any.host"