jenkins-build.py

#!/usr/bin/python -B

import sys
sys.dont_write_bytecode = True

import argparse
import os
import requests
import shutil
import subprocess

import build_tools


default_workspace = os.environ.get(
    'WORKSPACE',
    os.path.abspath(os.path.dirname(__file__)))

default_build_number = os.environ.get(
    'BUILD_NUMBER',
    '1337')

repos_failing_tests = []


def get_version(workspace):
    # load up the version number
    version_file = open(os.path.join(workspace, 'mb_version'), 'r')
    version = version_file.readline()
    version = version.strip()
    version_file.close()
    return version


def very_clean_repo(repo):
    # ignore repos not checked out yet
    if repo.exists():
        return repo.cmd(['git', 'clean', '-dfx'])


def update_sources(toolchain):
    # Quick hack because the jenkins builders don't seem to be able to create
    # this folder automatically as part of cloning?
    if not os.path.exists(build_tools.paths.toolchain_dir()):
        os.makedirs(build_tools.paths.toolchain_dir())
    print('\nUpdating toolchain...')

    with build_tools.utils.timer('updating sources'):
        rv = toolchain.switch_version()

    return rv


def parse_args(args, jenkins_parser):
    arg_parser = argparse.ArgumentParser(
        description='Builds an installer for the current platform',
        parents=(
            [build_tools.get_parent_parser(), jenkins_parser] +
            build_tools.builders.argument_parsers))

    arg_parser.add_argument(
        '-w',
        '--workspace',
        metavar='PATH',
        default=default_workspace,
        help='absolute path of Toolchain-Release')
    arg_parser.add_argument(
        '-n',
        '--build-number',
        metavar='N',
        default=default_build_number)
    arg_parser.add_argument(
        '-C',
        '--very-clean-build',
        action='store_true',
        help='use git to clean absolutely everything from each repo')
    arg_parser.add_argument(
        '--upload-docs',
        action='store_true',
        help='Upload docs to docs.soft.makerbot.net, implies --docs')
    arg_parser.add_argument(
        '--no-clean',
        action='store_true',
        help='skip some of the basic cleaning that would normally happen')
    arg_parser.add_argument(
        '--skip-update',
        action='store_true',
        help='skip update before building')
    arg_parser.add_argument(
        '--installer-only',
        action='store_true',
        help='assume the repos have been built, just packaging scripts')
    arg_parser.add_argument(
        '--publish-artifacts',
        action='store_true',
        help='publish artifacts from this build')
    arg_parser.add_argument(
        '--no-revs',
        dest='revs',
        action='store_false',
        help="don't write a revs file")

    if build_tools.utils.is_windows:
        angle_help = 'override the OpenGL version for qt with an angle version'
    else:
        # We should use argparse.SUPPRESS here, but it seems to cause
        # assertion errors in argparse. (Looks like it's adding a space
        # when it shouldn't and then checking that the thing with and
        # without the space are the same. They're not.)
        angle_help = 'Only used on windows'
    arg_parser.add_argument(
        '--use-angle',
        action='store_true',
        help='override the OpenGL version for qt with an angle version')

    return arg_parser.parse_args(args)


def path_for_toolchain(toolchain):
    """Find the path to the jenkins module for building a given toolchain"""
    if toolchain == build_tools.toolchain_list.desktop:
        if 'darwin' == sys.platform:
            jenkins_path = 'Bundle-Mac'
        elif 'win32' == sys.platform:
            jenkins_path = 'Bundle-Win'
        elif sys.platform.startswith('linux'):
            import platform
            (distro, version, codename) = platform.linux_distribution()
            if distro in ['Ubuntu']:
                jenkins_path = 'Bundle-Debian'
            elif distro in ['Fedora']:
                jenkins_path = 'Bundle-Redhat'
            else:
                raise ValueError(distro)
        else:
            raise ValueError(sys.platform)
    elif toolchain == build_tools.toolchain_list.argo:
        if 'win32' == sys.platform:
            jenkins_path = 'Bundle-Win'
        else:
            raise ValueError(sys.platform)
    elif toolchain == build_tools.toolchain_list.morepork:
        jenkins_path = 'Bundle-Morepork'
    else:
        raise Exception('toolchain not supported by jenkins-build.')

    return jenkins_path


def setup_build_args(all_args, toolchain):
    # handle the build arguments
    build_args = build_tools.builders.extract_build_args(all_args)

    # override certain args
    build_args['install'] = True

    # special case for SCons asks things to depend on the installed
    # versions of the other repos, rather than as siblings
    if 'scons_args' not in build_args:
        build_args['scons_args'] = []
    build_args['scons_args'].append('--no-devel-libs')

    if build_tools.utils.is_windows:
        # special case for windows toggle between console and windowed app
        # I would have come up with some special way to stick this in
        # the windows jenkins, but we're probably going to rip this out
        # in the not-too-distant future.
        build_args['scons_args'].append('--hide-console')

    # turn up the verbosity on cmake tests
    if 'cmake_test_args' not in build_args:
        build_args['cmake_test_args'] = []
    build_args['cmake_test_args'].append('-V')

    # we want --upload-docs to imply the --docs flags to ensure that
    # that docs are built when we try to upload them
    if build_args['upload_docs'] and 'docs' not in build_args:
        build_args['docs'] = True

    for source in toolchain:
        source.builder.bind_args(**build_args)


def upload_artifact(repo):
    rv = repo.upload(stable=True)
    if not rv:
        print("publish artifacts failed, passing")
    return rv


def build(fake_root, toolchain, additional_args):
    '''
        @param fake_root the install destination of the build
        @param toolchain Toolchain object representing the repo list
        @param additional_args, any extra building args
        @return True on success. Terminates script on failure
    '''

    if build_tools.utils.is_windows and additional_args.use_angle:
        # Hack to make mw-scons-tools work with the angle/opengl versions of QT
        os.putenv('QT5_ANGLE', 'TRUE')

    if os.path.exists(fake_root):
        shutil.rmtree(fake_root)

    result = build_tools.builders.SUCCESS

    for repo in toolchain:
        rv = repo.build()
        if rv is build_tools.builders.FAILURE:
            print('building ' + repo.name + ' failed')
            return rv
        elif rv is build_tools.builders.FAILED_TESTS:
            result = build_tools.builders.FAILED_TESTS
            repos_failing_tests.append(repo.name)
            # Sort of a hack, just do the install if the tests
            # fail. I think we only want this behaviour on
            # jenkins, otherwise I'd put this in builders.py
            repo.build(build=False, test=False, clean=False)
        # this being in an else block makes it so that we don't upload when
        # a repo fails to build or fails tests
        elif additional_args.publish_artifacts:
            if not upload_artifact(repo):
                repos_failing_tests.append(repo.name + "_artifacts")
                result = build_tools.builders.FAILED_TESTS
    return result


def run(args=[]):
    # We break arg parsing into two stages, because the initial
    # toolchain selection can affect further argument parsing
    toolchain_args, unparsed_args = (
        build_tools.get_parent_parser().parse_known_args(args))
    toolchain = build_tools.get_toolchain(toolchain_args)

    # Load the appropriate jenkins file
    jenkins_path = path_for_toolchain(toolchain)
    sys.path.insert(0, os.path.abspath(
        os.path.join(os.path.dirname(__file__), jenkins_path)))
    import jenkins

    # finish parsing
    arguments = parse_args(unparsed_args, jenkins.argument_parser())

    # hack to propogate platform because the platform gets set to default
    # which can be (and is) wrong sometimes
    arguments.platform = toolchain_args.platform

    # we have to do the default manually here because the
    # build argparser defaults most things to None
    if arguments.install_dir is None:
        # TODO(ted): pull from builders.py
        arguments.install_dir = os.path.join(arguments.workspace,
                                             "Install")

    version = '.'.join(
        [get_version(arguments.workspace), arguments.build_number])

    if arguments.very_clean_build:
        # should we be cleaning distributables?
        toolchain.for_each_repo(very_clean_repo)
        # no reason to clean if we just very cleaned
        arguments.clean = False

    if not arguments.no_clean and not arguments.installer_only:
        # clean the fake root
        if os.path.exists(arguments.install_dir):
            shutil.rmtree(arguments.install_dir, ignore_errors=True)

    # update (if applicable)
    if not arguments.skip_update and not arguments.installer_only:
        rv = update_sources(toolchain)
        if rv != 0:
            print('Updating failed!')
            return rv

    # jenkins checks out a commit hash when building,
    # however we want to upload it to artifactory with the branch
    # this is kind of hacky, and we make sure to do it after updating
    if 'Toolchain-Release' in toolchain:
        branch = os.environ.get('GIT_BRANCH')
        if branch:
            # jenkins stores the branch as `origin/[branch]`
            # so basename should drop the `origin/` part
            toolchain.toolchain_release.version = os.path.basename(branch)

    setup_build_args(arguments, toolchain)

    if arguments.revs:
        import json
        info = build_tools.build_info(toolchain)
        with open('revs', 'w') as revs:
            json.dump(info, revs, indent=2)

        print('')
        for source_dict in info:
            # HACKS: do all the hacky things!
            # we make some modifications to print more nicely in jenkins
            if 'modified' in source_dict and source_dict['modified']:
                source_dict['modified'] = ' (modified)'
            else:
                source_dict['modified'] = '\t'

            for arg in source_dict['build_args'].keys():
                # strip empty lists
                if source_dict['build_args'][arg] == []:
                    del source_dict['build_args'][arg]

            # Makes Distributables print more useful version info
            if 'versions' in source_dict:
                source_dict['version'] = (
                    source_dict['versions'][source_dict['version']])

            print('{name}:\t{version}{modified}\t{builder}'
                  '({build_args})'.format(**source_dict))

    print('')
    # We frequently benefit from knowing the env for a broken build
    print('Environment (parts may be overridden in code after this point):')
    print('')
    for k, v in sorted(os.environ.iteritems(), key=lambda t: t[0]):
        print('{}: {}'.format(k, v))
    print('')

    if not arguments.installer_only:
        with build_tools.utils.timer('build'):
            rv = build(fake_root=arguments.install_dir,
                       toolchain=toolchain,
                       additional_args=arguments)
            if rv is build_tools.builders.FAILED_TESTS:
                rv = build_tools.utils.mark_build_unstable(arguments.workspace)
                if rv != 0:
                    return -1
            elif rv is not build_tools.builders.SUCCESS:
                return -1

    with build_tools.utils.timer('Platform-specifc packaging'):
        # Actually do the platform-specific packaging
        rv = jenkins.run(
            workspace=arguments.workspace,
            version=version,
            toolchain=toolchain,
            fake_root=arguments.install_dir,
            additional_args=arguments)
        if rv is not build_tools.builders.SUCCESS:
            return -1

    if arguments.upload_docs:
        with build_tools.utils.timer('Publishing docs'):
            upload_url = 'makerbot@docs.soft.makerbot.net:repos/'
            upload_url += build_tools.paths.DEFAULT_REF + '/'
            copy_command = ['scp',
                            '-r',
                            os.path.join(arguments.install_dir, 'docs', '*'),
                            upload_url]
            subprocess.check_call(' '.join(copy_command), shell=True)

    if len(repos_failing_tests) > 0:
        for r in repos_failing_tests:
            print r + " is failing unit tests"
    return 0


if '__main__' == __name__:
    sys.exit(run(sys.argv[1:]))