IMPALA-11973: Add absolute_import, division to all eligible Python files

This takes steps to make Python 2 behave like Python 3 as
a way to flush out issues with running on Python 3. Specifically,
it handles two main differences:
 1. Python 3 requires absolute imports within packages. This
    can be emulated via "from __future__ import absolute_import"
 2. Python 3 changed division to "true" division that doesn't
    round to an integer. This can be emulated via
    "from __future__ import division"

This changes all Python files to add imports for absolute_import
and division. For completeness, this also includes print_function in the
import.

I scrutinized each old-division location and converted some locations
to use the integer division '//' operator if it needed an integer
result (e.g. for indices, counts of records, etc). Some code was also using
relative imports and needed to be adjusted to handle absolute_import.
This fixes all Pylint warnings about no-absolute-import and old-division,
and these warnings are now banned.

Testing:
 - Ran core tests

Change-Id: Idb0fcbd11f3e8791f5951c4944be44fb580e576b
Reviewed-on: http://gerrit.cloudera.org:8080/19588
Reviewed-by: Joe McDonnell <[email protected]>
Tested-by: Joe McDonnell <[email protected]>

bin/banned_py3k_warnings.txt

@@ -0,0 +1,2 @@
+no-absolute-import
+old-division

bin/dump_breakpad_symbols.py

@@ -53,6 +53,7 @@
 #   $IMPALA_TOOLCHAIN_PACKAGES_HOME/breakpad-*/bin/minidump_stackwalk \
 #   /tmp/impala-minidumps/impalad/03c0ee26-bfd1-cf3e-43fa49ca-1a6aae25.dmp /tmp/syms
 
+from __future__ import absolute_import, division, print_function
 import errno
 import logging
 import glob

bin/generate_minidump_collection_testdata.py

@@ -27,6 +27,7 @@
 # create the files in the interval [now - duration, now]. Minidumps are simulated by
 # making the files easily compressible by having some repeated data.
 
+from __future__ import absolute_import, division, print_function
 import errno
 import os
 import random

bin/get_code_size.py

@@ -19,7 +19,7 @@
 
 # This tool walks the build directory (release by default) and will print the text, data,
 # and bss section sizes of the archives.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import fnmatch
 import os
 import re

bin/inline_pom.py

@@ -21,7 +21,7 @@
 #
 # Usage: inline_pom.py <pom.xml>...
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import re
 import sys
 from tempfile import mkstemp

bin/load-data.py

@@ -20,7 +20,7 @@
 # This script is used to load the proper datasets for the specified workloads. It loads
 # all data via Hive except for parquet data which needs to be loaded via Impala.
 # Most ddl commands are executed by Impala.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import collections
 import getpass
 import logging

bin/parse-thrift-profile.py

@@ -39,7 +39,7 @@
 # 2018-04-13T15:06:34.144000 e44af7f93edb8cd6:1b1f801600000000 TRuntimeProfileTree(nodes=[TRuntimeProf...
 
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 from impala_py_lib import profiles
 import sys
 

bin/run-workload.py

@@ -27,7 +27,7 @@
 #   - Stores the execution details in JSON format.
 #
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import getpass
 import json
 import logging

bin/single_node_perf_run.py

@@ -69,7 +69,7 @@
 #   --start_minicluster   start a new Hadoop minicluster
 #   --ninja               use ninja, rather than Make, as the build tool
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 from optparse import OptionParser
 from tempfile import mkdtemp
 

bin/start-impala-cluster.py

@@ -20,6 +20,7 @@
 # Starts up an Impala cluster (ImpalaD + State Store) with the specified number of
 # ImpalaD instances. Each ImpalaD runs on a different port allowing this to be run
 # on a single machine.
+from __future__ import absolute_import, division, print_function
 import getpass
 import itertools
 import json
@@ -473,7 +474,7 @@ def compute_impalad_mem_limit(cluster_size):
   # memory choice here to max out at 12GB. This should be sufficient for tests.
   #
   # Beware that ASAN builds use more memory than regular builds.
-  physical_mem_gb = psutil.virtual_memory().total / 1024 / 1024 / 1024
+  physical_mem_gb = psutil.virtual_memory().total // 1024 // 1024 // 1024
   available_mem = int(os.getenv("IMPALA_CLUSTER_MAX_MEM_GB", str(physical_mem_gb)))
   mem_limit = int(0.7 * available_mem * 1024 * 1024 * 1024 / cluster_size)
   return min(12 * 1024 * 1024 * 1024, mem_limit)

docker/setup_build_context.py

@@ -20,6 +20,7 @@
 # Most artifacts are symlinked so need to be dereferenced (e.g. with tar -h) before
 # being used as a build context.
 
+from __future__ import absolute_import, division, print_function
 import argparse
 import glob
 import os

infra/python/bootstrap_virtualenv.py

@@ -32,7 +32,7 @@
 # This module can be run with python >= 2.7. It makes no guarantees about usage on
 # python < 2.7.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import glob
 import logging
 import optparse

lib/python/impala_py_lib/gdb/impala-gdb.py

@@ -19,7 +19,7 @@
 # A collection of useful Python GDB modules and commands for
 # debugging Impala core dumps.
 #
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import gdb
 from collections import defaultdict
 

lib/python/impala_py_lib/helpers.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 import fnmatch
 import logging
 import os

lib/python/impala_py_lib/profiles.py

@@ -18,7 +18,7 @@
 
 # This file contains library functions to decode and access Impala query profiles.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import base64
 import datetime
 import zlib

testdata/bin/check-hbase-nodes.py

@@ -20,6 +20,7 @@
 """Given a series of hosts and Zookeeper nodes, make sure that each node is accessible.
 """
 
+from __future__ import absolute_import, division, print_function
 import argparse
 import hdfs
 import logging
@@ -191,4 +192,4 @@ def is_hdfs_running(host, admin_user):
             LOGGER.error(msg)
             sys.exit(errors)
     else:
-        sys.exit(1)
+        sys.exit(1)

testdata/bin/generate-schema-statements.py

@@ -94,7 +94,7 @@
 # This should be used sparingly, because these commands are executed
 # serially.
 #
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import collections
 import csv
 import glob

testdata/bin/generate-test-vectors.py

@@ -40,7 +40,7 @@
 # The pairwise generation is done using the Python 'AllPairs' module. This module can be
 # downloaded from http://pypi.python.org/pypi/AllPairs/2.0.1
 #
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import collections
 import csv
 import math

testdata/bin/load-tpc-kudu.py

@@ -22,7 +22,7 @@
 # Kudu tables are created in the specified 'target-db' using the existing HDFS tables
 # from 'source-db'.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import logging
 import os
 import sqlparse

testdata/bin/load_nested.py

@@ -20,6 +20,7 @@
 '''This script creates a nested version of TPC-H. Non-nested TPC-H must already be
    loaded.
 '''
+from __future__ import absolute_import, division, print_function
 import logging
 import os
 

testdata/bin/random_avro_schema.py

@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 from random import choice, randint, random, shuffle
 from os.path import join as join_path
 from optparse import OptionParser

testdata/bin/rewrite-iceberg-metadata.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import glob
 import json
 import os

testdata/bin/wait-for-hiveserver2.py

@@ -22,7 +22,7 @@
 # TODO: Consider combining this with wait-for-metastore.py. A TCLIService client
 # can perhaps also talk to the metastore.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import os
 import time
 import getpass

testdata/bin/wait-for-metastore.py

@@ -21,7 +21,7 @@
 # to execute the get_database("default") Thrift RPC until the call succeeds,
 # or a timeout is reached.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import os
 import time
 from optparse import OptionParser

testdata/common/cgroups.py

@@ -19,7 +19,7 @@
 
 # Utility code for creating cgroups for the Impala development environment.
 # May be used as a library or as a command-line utility for manual testing.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import os
 import sys
 import errno

testdata/common/text_delims_table.py

@@ -22,7 +22,7 @@
 # command line, will generate data files in the specified directory and a
 # print a SQL load statement to incorporate into dataload SQL script generation.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 from shutil import rmtree
 from optparse import OptionParser
 from contextlib import contextmanager
@@ -35,7 +35,7 @@
 parser.add_option("--file_len", dest="file_len", type="int")
 
 def generate_testescape_files(table_location, only_newline, file_len):
-  data = ''.join(["1234567890" for _ in xrange(1 + file_len / 10)])
+  data = ''.join(["1234567890" for _ in xrange(1 + file_len // 10)])
 
   suffix_list = ["\\", ",", "a"]
   if only_newline:

testdata/common/widetable.py

@@ -22,7 +22,7 @@
 # generate a CSV data file and prints a SQL load statement to incorporate
 # into dataload SQL script generation.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 from datetime import datetime, timedelta
 import itertools
 import optparse
@@ -51,7 +51,7 @@ def get_columns(num_cols):
   iter = itertools.cycle(templates)
   # Produces [bool_col1, tinyint_col1, ..., bool_col2, tinyint_col2, ...]
   # The final list has 'num_cols' elements.
-  return  [iter.next() % (i / len(templates) + 1) for i in xrange(num_cols)]
+  return [iter.next() % (i // len(templates) + 1) for i in xrange(num_cols)]
 
 # Data generators for different types. Each generator yields an infinite number of
 # value strings suitable for writing to a CSV file.

tests/authorization/test_authorization.py

@@ -17,6 +17,7 @@
 #
 # Client tests for SQL statement authorization
 
+from __future__ import absolute_import, division, print_function
 import os
 import pytest
 import tempfile

tests/authorization/test_authorized_proxy.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 import pytest
 import os
 import grp

tests/authorization/test_provider.py

@@ -17,6 +17,7 @@
 #
 # Client tests for SQL statement authorization
 
+from __future__ import absolute_import, division, print_function
 import pytest
 import os
 import tempfile

tests/authorization/test_ranger.py

@@ -17,6 +17,7 @@
 #
 # Client tests for SQL statement authorization
 
+from __future__ import absolute_import, division, print_function
 import os
 import grp
 import json

tests/beeswax/impala_beeswax.py

@@ -25,6 +25,7 @@
 #   client.connect()
 #   result = client.execute(query_string)
 #   where result is an object of the class ImpalaBeeswaxResult.
+from __future__ import absolute_import, division, print_function
 import logging
 import time
 import shlex
@@ -282,7 +283,7 @@ def __build_summary_table(self, summary, idx, is_fragment_root, indent_level,
             setattr(max_stats, attr, max(getattr(max_stats, attr), val))
 
       if len(node.exec_stats) > 0:
-        avg_time = agg_stats.latency_ns / len(node.exec_stats)
+        avg_time = agg_stats.latency_ns // len(node.exec_stats)
       else:
         avg_time = 0
 

tests/benchmark/plugins/clear_buffer_cache.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 from tests.util.cluster_controller import ClusterController
 from tests.benchmark.plugins import Plugin
 

tests/benchmark/plugins/vtune_plugin.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 from os import environ
 from tests.util.cluster_controller import ClusterController
 from tests.benchmark.plugins import Plugin

tests/benchmark/report_benchmark_results.py

@@ -28,8 +28,7 @@
 # be an int (2). The following line changes this behavior so that float will be returned
 # if necessary (2.5).
 
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import difflib
 import json
 import logging

tests/catalog_service/test_catalog_service_client.py

@@ -17,6 +17,7 @@
 #
 # Tests to validate the Catalog Service client APIs.
 
+from __future__ import absolute_import, division, print_function
 import logging
 import pytest
 

tests/catalog_service/test_large_num_partitions.py

@@ -18,6 +18,7 @@
 # Tests to validate the Catalog Service works properly when partitions
 # need to be fetched in multiple batches.
 
+from __future__ import absolute_import, division, print_function
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.test_dimensions import create_single_exec_option_dimension
 

tests/common/base_test_suite.py

@@ -16,6 +16,7 @@
 # under the License.
 
 # The base class that should be used for tests.
+from __future__ import absolute_import, division, print_function
 import logging
 
 from tests.common.test_vector import ImpalaTestMatrix

tests/common/custom_cluster_test_suite.py

@@ -18,6 +18,7 @@
 # Superclass for all tests that need a custom cluster.
 # TODO: Configure cluster size and other parameters.
 
+from __future__ import absolute_import, division, print_function
 import logging
 import os
 import os.path

tests/common/environ.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 import json
 import logging
 import os

tests/common/file_utils.py

@@ -19,6 +19,7 @@
 # and other functions used for checking for strings in files and
 # directories.
 
+from __future__ import absolute_import, division, print_function
 import os
 import re
 import tempfile

tests/common/iceberg_test_suite.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import, division, print_function
 import datetime
 
 from tests.common.impala_test_suite import ImpalaTestSuite