sis

#!/usr/bin/env python
from copy                    import deepcopy
from datetime                import datetime
from glob                    import iglob as glob
from urllib                  import urlopen
import argparse
import imp
import importlib
import logging
import multiprocessing
import os
import sys
import traceback

from sisyphus.test.test                import ensure_dir, Environment
from sisyphus.test.suite               import run_argparser_setups, testsuites
from sisyphus.util.concurrent.futures  import ThreadPoolExecutor
from sisyphus.util.parallelism         import available_cpu_count
import sisyphus

_version        = sisyphus.VERSION
_report_name    = "stats-" + datetime.now().strftime("%Y.%m.%d")
_out            = sys.stdout

_default_environment = Environment(
    arch_dirs        = [],
    compile_times    = False,
    builddir         = "build/",
    debug            = False,
    default_dirs     = [],
    expect_url       = "",
    faillog_out      = None,
    reportdir        = "reports/",
    runexe           = "",
    show_disappeared = False,
    threads          = available_cpu_count() + 1,
    verbose          = False,
    max_fails        = 0,
)

_argparser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS)
_argparser.add_argument(dest="testfilter", metavar="TEST", nargs="*",
                        help="only run test TEST")
_argparser.add_argument("-d", "--debug", dest="debug", action="store_true",
                        help="Enable debug messages for sisyphus")
_argparser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                        help="More output, e.g. expected test results")
_argparser.add_argument("--show-fail-output", dest="show_output", action="store_true",
                        help="Show stdout/stderr for unexpected failures")
_argparser.add_argument("-c", "--compile-times", dest="compile_times",
                        action="store_true",
                        help="Display compile time of each program")
_argparser.add_argument("-t", "--threads", dest="threads", type=int,
                        help="Number of threads to use")
_argparser.add_argument("--max-expect-fails", dest="max_fails", type=int,
                        help="If this many expectations fail, stop")
_argparser.add_argument("--faillog", dest="faillog_out",
                        help="Write faillog into file specified here")
_argparser.add_argument("--expect", dest="expect_url",
                        help="file/url with fail expectations")
_argparser.add_argument("--show-disappeared", dest="show_disappeared",
                        action="store_true", help="show disappeared tests")
_argparser.add_argument("--version", action="version", version=_version)
_argparser.add_argument("-s", "--suite", dest="suite", action='append',
                        help="select testsuite to run")

# List of (matchfunc, factory) tuples.
# if matchfunc(filename) returns True then factory(filename, environment) will
# be used to construct a Test for this file.
_test_factories = []

_console_red    = "\033[1;31m"
_console_green  = "\033[1;32m"
_console_yellow = "\033[1;33m"
_console_bold   = "\033[1m"
_console_normal = "\033[m"
_console_clear  = "\r\033[K"
_rc             = 0  # application return code
if not sys.stdout.isatty():
    _console_red    = ""
    _console_green  = ""
    _console_yellow = ""
    _console_bold   = ""
    _console_normal = ""
    _console_clear  = ""

_LOGGER = logging.getLogger("sisyphus")

class TooManyFails(Exception):
    pass

def load_expectations(url, environment):
    try:
        fetched = True
        input   = urlopen(url)
        if input.getcode() is not None and input.getcode() != 200:
            _out.write("Warning: Download '%s' fail. status code: %d\n"
                       % (url, input.getcode()))
            fetched = False
    except Exception as e:
        _out.write("Warning: Download '%s' fail: %s\n"
                   % (url, e))
        fetched = False

    cachefile = os.path.join(environment.builddir, os.path.basename(url))
    if not fetched:
        _out.write("Warning: Using cached expect file\n")
        input = open(cachefile)

    results = dict()
    for line in input:
        line = line.decode("utf-8")
        try:
            i = line.index(" ")
        except ValueError:
            continue
        test_id          = line[:i]
        result           = line[i:].strip()
        results[test_id] = result

    if fetched:
      try:
        ensure_dir(environment.builddir)
        cache = open(cachefile, "w")
        for (test_id, result) in results.iteritems():
            cache.write("%-40s %s\n" % (test_id, result))
        cache.close()
      except Exception as e:
        _out.write("Warning: Writing cache file '%s' failed: %s\n"
                   % (cachefile, e))

    return results


def expectation_match(error, expectation):
    if error.endswith(" compile errors") and \
       expectation.endswith(" compile errors"):
        # exact number of compile errors does not matter
        return True
    if error == "compile: SIGXCPU" and expectation == "compile: SIGKILL" or \
       error == "execute: SIGXCPU" and expectation == "execute: SIGKILL":
        # BSD on timeout signals SIGXCPU and Linux SIGKILL
        return True
    if error == "compile: SIGBUS" and expectation == "compile: SIGSEGV" or \
       error == "execute: SIGBUS" and expectation == "execute: SIGSEGV":
        # OS X often signals SIGBUS where others signal SIGSEGV
        return True
    return error == expectation


def console_output(environment, test, expected, have_expectations):
    timing = ""
    prefix = ""
    if environment.compile_times:
        time = None
        if hasattr(test, "stepresults") and "compile" in test.stepresults:
            result = test.stepresults["compile"]
            if hasattr(result, "time"):
                time = result.time
        if time is not None:
            timing = " [%s%.2fs%s]" % (_console_yellow, time, _console_normal)
    result = test.result
    if expected and expectation_match(result, expected):
        if test.success and not environment.verbose:
            newline = "\n" if _console_clear == "" else ""
            _out.write("%s%s...%s" % (_console_clear, test.id, newline))
            _out.flush()
            return
    else:
        if test.success:
            prefix = _console_green
        elif expected and expected != "ok":
            prefix = _console_yellow
        else:
            prefix = _console_red

        if expected and not test.success:
            global _rc
            _rc = 1

        if expected:
            result += " (expected %s)" % expected
        elif have_expectations:
            result += " (new)"
    _out.write("%s%s%-40s %s%s%s\n" % (_console_clear, prefix, test.id, result, _console_normal, timing))
    _out.flush()

def print_output_if_failed(env, test, expected):
    if not hasattr(env, "show_output"):
        return
    elif not env.show_output:
        return
    if expected and expectation_match(test.result, expected):
        return
    if test.stdout:
        print "-"*10, "stdout:", "-"*10
        sys.stdout.write(test.stdout)
    if test.stderr:
        print "-"*10, "stderr:", "-"*10
        sys.stdout.write(test.stderr)
    if test.diff:
        print "-"*10, "diff:  ", "-"*10
        sys.stdout.write(test.diff)
    if test.stdout or test.stderr or test.diff:
        print "-"*10, "end out", "-"*10
    else:
        _LOGGER.debug("-- no output --")

class Report:
    def __init__(self):
        self.tests   = list()
        self.summary = (0, 0)

    def add_test(self, test):
        self.tests.append(test)
        fail = not test.success
        self.summary = (self.summary[0]+1, self.summary[1]+int(fail))

    def print_summary(self):
        _out.write("%s---------------------------\n" % (_console_clear))
        _out.write("Ran %d tests, of which %s%d failed%s.\n" %
                   (self.summary[0], _console_bold, self.summary[1], _console_normal))
        _out.flush()

    def write_faillog(self, fh):
        for test in self.tests:
            fh.write("%-40s %s\n" % (test.id, test.result))


def run_test(test, expect_result):
    try:
        test.run(expect_result)
    except Exception as e:
        (_, _, tb) = sys.exc_info()
        _LOGGER.error("Exception in %s: %s\n%s" % (test.id, e, traceback.format_exc(tb)))
        raise e
    return test


def makereport(environment, tests):
    ensure_dir(environment.reportdir)

    expectations = {}
    have_expectations = False
    if environment.expect_url:
        try:
            expectations = load_expectations(environment.expect_url, environment)
            have_expectations = True
        except Exception as e:
            _out.write("Couldn't load fail expectations: %s\n" % e)

    # create test futures for parallel evaluation
    queue    = list()
    _LOGGER.debug("run thread pool with %d workers max" % environment.threads)
    executor = ThreadPoolExecutor(max_workers=environment.threads)
    for test in tests:
        expect_result = "ok"
        if have_expectations:
            expect_result = expectations.get(test.id, "ok")
        queue.append(executor.submit(run_test, test, expect_result))

    # collect report
    r = Report()
    expect_fail_count = 0
    try:
        found = {}
        for promise in queue:
            test           = promise.result()
            found[test.id] = True
            expected       = expectations.get(test.id)
            console_output(environment, test, expected, have_expectations)
            r.add_test(test)
            print_output_if_failed(environment, test, expected)
            # abort if we unexpectedly fail too often
            if expected and not test.success\
                        and not expectation_match(test.result, expected)\
                        and environment.max_fails > 0:
                expect_fail_count += 1
                if expect_fail_count >= environment.max_fails:
                    raise TooManyFails()
        if environment.show_disappeared:
            for t in expectations:
                if t not in found:
                    _out.write("%s%-40s %s%s\n" % (_console_red, t, "test disappeared", _console_normal))
                    _out.flush()
    except KeyboardInterrupt:
        _out.write("Received interrupt signal, shutting down\n")
    except TooManyFails:
        _out.write("More than %d expectations failed, shutting down\n" % environment.max_fails)
    finally:
        for promise in queue:
            promise.cancel()
        executor.shutdown(wait=False)
    faillog_out = environment.faillog_out
    if faillog_out is None:
        faillog_out = environment.reportdir + "/" + _report_name + ".faillog"
    r.write_faillog(open(faillog_out, 'w'))
    r.print_summary()

def warn_about_bashism_in_path():
    path = os.environ.get("PATH")
    if "~" in path:
        _LOGGER.info("Warning: '~' in PATH is only expanded in the bash shell (not in sisyphus)")

def main():
    global _rc
    os.putenv("LC_ALL", "C")  # normalize environment
    logging.basicConfig(
        level=logging.ERROR,
        format="%(relativeCreated)d %(levelname)s: %(message)s")
    _LOGGER.setLevel(logging.WARNING)

    # do not clutter our plugin directories with .pyc files...
    sys.dont_write_bytecode = True

    # Load "default" testsuite if available in the current directory
    default_testsuite_filename = "./testsuite.py"
    if os.path.exists(default_testsuite_filename):
        if sys.path[0] != "":
            sys.path.insert(0, "")
        try:
            importlib.import_module("testsuite")
        except Exception as e:
            (_, _, tb) = sys.exc_info()
            _LOGGER.error("Loading default testsuite '%s' failed: %s\n%s" % (default_testsuite_filename, e, traceback.format_exc(tb)))


    # Setup environment/argparsers from other modules
    _default_environment.may_override = False
    run_argparser_setups(_argparser, _default_environment)
    _default_environment.may_override = True

    # Parse arguments
    args = Environment()
    _argparser.parse_args(namespace=args)
    # Make sure that the user did not use default values in argparse,
    # they are not compatible with the hierarchical nature of sisyphus
    for (key,value) in args.items():
        if value == _argparser.get_default(key):
            _LOGGER.error("Default value in argparse argument '%s' will override testsuite environment!" % key)

    # Select testsuites/construct testlist
    tests = []
    if hasattr(args, "suite"):
        for suite in args.suite:
            testsuite = None
            for s in testsuites:
                if s.name == suite:
                    testsuite = s
                    break
            if testsuite is None:
                _LOGGER.error("Testsuite '%s' does not exist" % suite)
                continue
            for test in testsuite.tests:
                test.testsuite = testsuite
            tests += testsuite.tests
    else:
        had_default_testsuites = False
        for testsuite in testsuites:
            if not testsuite.default:
                continue
            had_default_testsuites = True
            for test in testsuite.tests:
                test.testsuite = testsuite
            tests += testsuite.tests
        if not had_default_testsuites:
            suites = ", ".join(s.name for s in testsuites)
            _LOGGER.error("No default. Select testsuite from:\n"+suites)

    # Filter testlist if necessary
    if hasattr(args, "testfilter") and len(args.testfilter) > 0:
        testmap = dict()
        for test in tests:
            testmap[test.id] = test
        newtests = []
        for filter in args.testfilter:
            test = testmap.get(filter)
            if test is None:
                # see if any test has the string as prefix/suffix
                found_one = False
                for test in tests:
                    if test.id.startswith(filter) or test.id.endswith(filter):
                        newtests.append(test)
                        found_one = True
                if not found_one:
                    _LOGGER.warning("Unknown test '%s'" % filter)
                    continue
            else:
                newtests.append(test)
        tests = newtests

    # Sort tests
    tests = sorted(tests, key=lambda x: x.id)

    # Prepare test environments
    base_environment = Environment()
    base_environment.merge(_default_environment)
    for test in tests:
        old_testenv = test.environment
        newenv = Environment()
        newenv.merge(_default_environment)
        newenv.merge(test.testsuite.environment)
        # Include testsuite environment into global environment
        # TODO: broken if multiple testsuites are selected
        base_environment.merge(test.testsuite.environment)
        newenv.merge(old_testenv)
        newenv.merge(args)
        test.environment = newenv
    base_environment.merge(args)
    if base_environment.verbose:
        _LOGGER.setLevel(logging.INFO)
    if base_environment.debug:
        _LOGGER.setLevel(logging.DEBUG)

    warn_about_bashism_in_path()

    # Run the tests
    makereport(base_environment, tests)
    return _rc

if __name__ == "__main__":
    retcode = main()
    sys.exit(retcode)