master.cfg

# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# ex: set syntax=python:
# vim: ft=python
# NIpy buildBOT Master Instance config
#
# [ TODO ]      (in no particular order)
# - combine multiple steps into one (if possible)
# - rename steps to be more concise and informative
# - generate numpy version based templates
# - what's up with the buildout not using ~/.buildout/default.cfg and not caching
#       * we may need to set some properties for this to work
# - install more versions of python on the slaves
# - have slaves report back their configuration (so we have them doing the right builders)
# - if possible, forcebuild every builder on a particular slave (to check that it can)
# - hookup with git post-commit hooks (instead of relying on poller)

""" A brief tour of buildbot

To remind myself

A ChangeSource looks for Changes.

In our case we are using GitPoller ChangeSource to look for changes in our git
repositories.

When ChangeSource finds changes, it broadcasts them to all Schedulers.  Each
Scheduler must decide if it wants to react to a Change.  It generally does this
by filtering changes to select changes it will act on.

We use SingleBranchSchedulers for this.  These obviously respond only to
changes in one branch of a repository. .schedulers() outputs list of
SingleBranchSchedulers with each linked to builders corresponding to each
branch if there are more than one.

A BuildSet is the association of (information about what changes need to be
tested) and (Builders on which to test)

If a Scheduler selects changes for action, it sends BuildSets to the system
(the BuildMaster).  Therefore, a Scheduler needs to know its Builders. It gets
these by name (string).

For each BuildSet, the (changes to be tested) are sent to each Builder.  Each
of these requests is BuildRequest.

A Builder accepts a BuildRequest and runs it on a BuildSlave.  It does this by
using its own BuildFactory instance to make Builds.

A Builder can have more than one associated BuildSlave.  In this case each
associated BuildSlave is assumed to produce identical results, and the Builder
can therefore choose at whim which BuildSlave to use.

A Build has one or more Steps that should be run with a given (changes to be
tested).

One type of build Step is a Trigger, which activates a triggerable Scheduler,
which can react any way it likes, including sending further BuildSets.
"""

import re

# This is convenient for splitting sh commands into lists
import shlex

# Buildbot objects
from buildbot.locks import MasterLock
from buildbot.buildslave import BuildSlave
from buildbot.changes.gitpoller import GitPoller
from buildbot.schedulers.filter import ChangeFilter
from buildbot.schedulers.forcesched import (ForceScheduler,
                                            StringParameter)
from buildbot.schedulers.basic import (SingleBranchScheduler)
from buildbot.schedulers import timed
from buildbot.schedulers.triggerable import Triggerable
from buildbot.schedulers.trysched import Try_Jobdir
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand, SetPropertyFromCommand
from buildbot.steps.transfer import FileDownload, DirectoryUpload
from buildbot.steps.trigger import Trigger
from buildbot.process.properties import Property, Interpolate
from buildbot.config import BuilderConfig

HTTP_SERVER_URL='https://nipy.bic.berkeley.edu/scipy_installers/'

# Directory for cached downloads of files needed by buildbots
HTTP_CACHE_URL = HTTP_SERVER_URL + 'buildbot-files/'

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

"""
Load local setup from python module
Local setup is:

* master_url : str; URL that the master serves on.  Putting this in the setup
  allows us to run master on another machine for debugging
* slave_portnum : int; slave port number, as above
* slave_passes : dict; key, value pairs of (slave name, password) giving slave
  names and slave password.

Don't store local_setup in version control with real passwords !  Use
secret_passwords.py instead, and don't put in version control.
"""
import local_setup
reload(local_setup) # so buildbot reconfig refreshes these parameters

# pip installs that should always be '--ignore-installed'
PIP_IGNORE_INSTALLS = ('nose', 'sphinx')

# Check parameters in setup
for attr_name in ('slave_passes', 'master_url', 'slave_port'):
    assert hasattr(local_setup, attr_name)

# Fetch any secret passwords.  We're using this trick so it is harder to save
# the secret passwords accidentally into version control
try:
    import secret_passwords
    reload(secret_passwords)
except ImportError:
    pass
else:
    for slavename, password in secret_passwords.slave_passes.items():
        local_setup.slave_passes[slavename] = password

# Check that slave names are unique
if len(set(local_setup.slave_passes)) != len(local_setup.slave_passes):
    raise ValueError('Slave names must be unique')

####### BUILDSLAVES

# Slave groups
linux_64_slaves = ['tom-standard']
debian_ppc_slaves = ['debian-ppc-64']
sparc_slaves_sid = ['nd-bb-slave-sparc-sid', 'nd-bb-slave-sparc64-sid']
sparc_slaves_wheezy = ['nd-bb-slave-sparc-wheezy', 'nd-bb-slave-sparc64-wheezy']
sparc_slaves = ['nd-bb-slave-sparc']
osx_tiger_ppc_slaves = ['osx-mini']
osx_snowleopard_slaves = ['osx-10.6']
osx_lion_slaves = ['osx-10.7']
osx_yosemite_slaves = ['mac-pro']
win7_32_slaves = ['win7-32-mini']
win7_64_slaves = ['win-2008-64']
win10_64_slaves = ['win10']
win2016_slaves = ['win2016']
arm_slaves = ['arm-raspberry-pi']
fedora_slaves = ['i20-fedora']
osx_wheel_slaves = ['osx-10.6-clean']

# Locks to prevent resource exhaustion which might need to be defined
# for some slaves.  It seems that locks need to be given to BuildSlave
# at __init__ time, so let's construct the mapping first: slave -> [locks]
# which are later used while initiating BuildSlave's
def add_lock(slave_locks, slavenames, lock, *accessargs):
    """Add a LockAccess's to a lock to the list of slaves
    """
    for name in slavenames:
        if not (name in slave_locks):
            slave_locks[name] = []
        slave_locks[name].append(lock.access(*accessargs))

slave_locks = {}

# vagus sparc box has only 4 CPUs, thus we should not run more than 4
# jobs in parallel
add_lock(slave_locks,
         ['nd-bb-slave-sparc'+suf for suf in ['', '-wheezy', '-sid']],
         MasterLock("vagus", maxCount=4), 'counting')

# lingual sparc box has only 2 CPUs, thus we should not run more than 2
# jobs in parallel
add_lock(slave_locks,
         ['nd-bb-slave-sparc64'+suf for suf in ['-wheezy', '-sid']],
         MasterLock("lingual", maxCount=2), 'counting')

# wheel slave does awful things to /usr/local, should only run one job at a
# time
add_lock(slave_locks,
         osx_wheel_slaves,
         MasterLock("wheel-usr-local", maxCount=1), 'counting')

# These are working desktops, so throttle them to keep the machines calm
add_lock(slave_locks,
         ['tom-standard'],
         MasterLock("working-desktops", maxCount=1), 'counting')

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
slaves = [BuildSlave(slave_name, slave_pass, locks=slave_locks.get(slave_name, None))
    for slave_name, slave_pass in local_setup.slave_passes.items()]

# slave names must be unique
c['slaves'] = slaves

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = local_setup.slave_port

# Helper functions and constants
def sanitize_branch_name(name):
    return name.replace('/', '+')

def is_one_default_branch(branches, branch):
    return (len(branches) < 2 and (branch in (None, 'master', 'HEAD')))

def get_branch_builder_name(builder, branches, branch=None):
    if branch is None or is_one_default_branch(branches, branch):
        return builder
    else:
        return '%s_%s' % (builder, sanitize_branch_name(branch))


def pyorg_pybin(pyver):
    return (
        '/Library/Frameworks/Python.framework/Versions/{0}/bin/python{0}'.format(
            pyver))


def needs_ignore_installed(pkg_spec):
    # True if packages in this `pkg_spec` need ``--ignore-installed``
    for spec in pkg_spec:
        pkg_name = re.split('[\[ =<>!,]*', spec)[0]
        if pkg_name in PIP_IGNORE_INSTALLS:
            return True
    return False


def do_pip_install(factory, pip_depends, env, pip_args=None):
    if pip_args is None:
        pip_args = []
    for pkg_spec in pip_depends:
        if isinstance(pkg_spec, basestring):
            pkg_spec = [pkg_spec]
        cmd = ['pip', 'install'] + list(pip_args)
        # We sometimes need the 'ignore-installed' flag to force pip to install
        # the programs in the virtualenv.  Otherwise pip finds the programs
        # (like 'nose') already installed; these then call the system python
        # and we don't get the virtualenv python.
        if needs_ignore_installed(pkg_spec):
            cmd += ['--ignore-installed']
        factory.addStep(
            ShellCommand(command=cmd + list(pkg_spec), env=env))


def make_virtualenv(factory, python=None, install_pip=True,
                    sys_site_packages=False):
    # Obviously requires virtualenv on the slave
    # Because of the bug in elderly easy_install, it would take into
    # account settings present in setup.cfg and might try to install
    # under /usr/lib as happened with fail2ban bot.  So deploy virtualenv
    # while stepping into a new clean directory.
    venv_params = ['--system-site-packages'] if sys_site_packages else []
    venv_params += ['--clear', '../venv']
    if not python is None:
        venv_params = ['--python=' + python] + venv_params
    factory.addStep(ShellCommand(command=['virtualenv'] + venv_params,
                                    workdir='build/mk_venv'))
    # Now we need the venv binary directory
    factory.addStep(FileDownload('slave_scripts/bs_tools.py',
                                 'bs_tools.py'))
    factory.addStep(SetPropertyFromCommand(command=[
        'python', 'bs_tools.py', 'path-prepend-str', 'venv'],
        property='venv_script_dir'))
    factory.addStep(SetPropertyFromCommand(command=[
        'python', 'bs_tools.py', 'abspath', 'venv'],
        property='venv_dir'))
    # Set env, with virtualenv in front of the path
    env={'PATH': Interpolate("%(prop:venv_script_dir)s${PATH}")}
    workdir = Interpolate("%(prop:venv_dir)s")
    if install_pip:
        # Upgrade pip if necessary for wheel installs
        factory.addStep(ShellCommand(
            command=['pip', 'install', '--ignore-installed', 'pip>=6'],
            env=env))
    return env, workdir


def add_bs_tools(factory):
    # Put bs_tools into properties in case someone needs it
    factory.addStep(SetPropertyFromCommand(
        command = ['python', 'bs_tools.py', 'abspath', 'bs_tools.py'],
        property = 'bs_tools'))


class GithubBot(object):
    """ Class to make buildbot stuff based round a github repo
    """

    def __init__(self,
                 organization,
                 repo,
                 test_cmd,
                 doc_build_cmd=None,
                 built_doc_src=None,
                 pip_depends=(),
                 doc_pip_depends=('sphinx',),
                 src_subdir=None,
                 download_dir=None,
                 build_timeout=None,
                 test_env=None,
                 test_dir=None,
                 branches=["master"],
                 ):
        # using easy_install on packages that are already in
        # system-site-packages causes the system path to get promoted above the
        # virtualenv path, and usually causes a terrible mess.  so prefer pip
        # if at all possible.
        self.organization = organization
        self.repo = repo
        self.git_url = "git://github.com/%s/%s.git" % (organization, repo)
        self.test_cmd = test_cmd
        self.test_env = test_env
        self.test_dir = test_dir
        self.doc_build_cmd = doc_build_cmd
        self.built_doc_src = built_doc_src
        # Dependencies for install / test
        self.pip_depends = pip_depends # pip
        self.doc_pip_depends = doc_pip_depends # pip installs for docs
        self.src_subdir = src_subdir # Subdirectory containing source
        if download_dir is None:
            download_dir = '%s-dist' % repo
        self.download_dir = download_dir
        self.build_timeout = build_timeout
        self.branches = branches

    @property
    def project(self):
        return "%s" % (self.repo,)

    def poller(self, branches=None, interval=300):
        if branches is None:
            branches = self.branches
        return GitPoller(
            self.git_url,
            project=self.project,
            category=self.project,
            workdir = "%s-%s-%s" % (self.organization, self.repo, sanitize_branch_name('_'.join(branches))),
            branches=branches,
            pollinterval=interval)

    def _get_scheduler(self, branch_buildernames, branches, branch):
        # we need a closure to maintain the function comparing to the
        # actual target branch
        def _branch_fn(b):
            return lambda b_: b_==b
        branch_fn = _branch_fn(branch)
        return SingleBranchScheduler(
            name=get_branch_builder_name(self.project, branches, branch),
            change_filter=ChangeFilter(
                 repository=self.git_url,
                 project=self.project,
                 category=self.project,
                 branch_fn=branch_fn),
            treeStableTimer=None,
            builderNames=branch_buildernames)

    def schedulers(self, buildernames, branches=None):
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        if branches is None:
            branches = self.branches
        # Create a separate scheduler per each branch linking it
        # to correspondingly named builders
        return [self._get_scheduler(
                      [get_branch_builder_name(n, branches, branch)
                       for n in buildernames],
                      branches, branch)
                for branch in branches]

    def nightly_scheduler(self,
                          buildernames,
                          name_suff,
                          hour = 2, minute = 23,
                          ifchanged=True,
                          branch='master',
                         ):
        """ Make a timed scheduler for this repo """
        if isinstance(buildernames, basestring):
            buildernames = [buildernames]
        name = "%s-%s-nightly-%s" % (self.organization, self.repo, name_suff)
        changefilter = ChangeFilter(repository=self.git_url,
                                    project=self.project,
                                    category=self.project)
        return timed.Nightly(name=name,
                branch=branch,
                builderNames=buildernames,
                change_filter=changefilter,
                hour=hour,
                minute=minute,
                onlyIfChanged=ifchanged)

    def _code_build_factory(self,
                            python=None,
                            pip_depends=None,
                            pre_build_cmds=(),
                            branch=None,
                            install_pip=None):
        if pip_depends is None:
            pip_depends = self.pip_depends
        factory = BuildFactory()
        git_kwargs = {'branch': branch} if branch is not None else {}
        factory.addStep(Git(repourl=self.git_url,
                            mode='full',
                            retry=(15, 5),
                            **git_kwargs))
        # Make virtualenv, return environment, workdir of venv
        my_env, workdir = make_virtualenv(
            factory, python,
            install_pip=bool(install_pip or pip_depends),
            sys_site_packages=True)
        # Any pre-building commands
        for cmd in pre_build_cmds:
            factory.addStep(ShellCommand(command=cmd))
        # Add install and runtime dependencies
        do_pip_install(factory, pip_depends, my_env)
        # Show pip enviroment
        factory.addStep(ShellCommand(command=['pip', 'freeze'], env=my_env))
        # Work out working directory for install
        if not self.src_subdir is None:
            factory.addStep(SetPropertyFromCommand(command=[
                'python', 'bs_tools.py', 'abspath', self.src_subdir],
                property='src_path'))
            build_workdir = Interpolate("%(prop:src_path)s")
        else:
            build_workdir = None
        return factory, my_env, workdir, build_workdir

    def factory(self,
                test_cmd=None,
                python=None,
                pip_depends=None,
                pre_build_cmds=(),
                post_cmds=(),
                test_env=None,
                test_dir=None,
                install_cmd=('python', 'setup.py', 'install'),
                install_wd=None,
                branch=None,
        ):
        if test_cmd is None:
            test_cmd = self.test_cmd
        if test_env is None:
            test_env = self.test_env
        if test_dir is None:
            test_dir = self.test_dir
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, pip_depends, pre_build_cmds, branch=branch)
        # Additional options for ShellCommands, if provided.  To not
        # rely on knowing internal logic (either handles None as the
        # default value), we explicitly construct a dictionary only
        # with the specified options
        kwargs = {}
        if self.build_timeout:
            kwargs['timeout'] = self.build_timeout
        # Install into virtualenv
        if install_wd is None:
            install_wd = build_workdir
        if install_cmd is not None:
            factory.addStep(ShellCommand(
                command=install_cmd,
                env=env,
                workdir=install_wd,
                **kwargs))
        # Test
        if test_env:
            env = dict(env.items() + test_env.items())
        if test_dir:
           # allow for interpolations using known know variables
           test_dir_env = {'workdir': workdir, 'build_workdir': build_workdir}
           test_dir_env.update(env)
           test_dir = test_dir % test_dir_env
        else:
            test_dir = workdir
        if isinstance(test_cmd, dict):
            # must be mapping branches to test_cmd
            test_cmd = test_cmd[branch]
        factory.addStep(ShellCommand(command=test_cmd,
            env=env, workdir=test_dir, **kwargs))
        for cmd in post_cmds:
            factory.addStep(
                ShellCommand(command=shlex.split(cmd, posix=True),
                             env=env,
                             workdir=workdir,
                             **kwargs))
        return factory

    def bdist_factory(self,
                      bdist_type,
                      test_cmd=None,
                      python=None,
                      pip_depends=None,
                      pre_build_cmds=()):
        if test_cmd is None:
            test_cmd = self.test_cmd
        factory, env, workdir, build_workdir = self._code_build_factory(
            python, pip_depends, pre_build_cmds, install_pip=True)
        # Build binary format(s)
        if python is None: # Use virtualenv python
            my_python = 'python'
            build_env = env
        else: # Use specified python
            my_python = python
            # Insert virtualenv into PYTHONPATH for building using virtualenv
            # dependencies
            factory.addStep(SetPropertyFromCommand(command=shlex.split(
                'python bs_tools.py get-python-lib'),
                property='venv_python_lib', env=env))
            build_env = env.copy()
            build_env['PYTHONPATH'] = Interpolate("%(prop:venv_python_lib)s")
        if bdist_type == 'egg':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_egg'],
                env=build_env,
                workdir=build_workdir))
            # Make a mpkg for good measure
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist_mpkg'],
                env=build_env,
                workdir=build_workdir))
            installer = ['easy_install']
        elif bdist_type == 'exe':
            factory.addStep(ShellCommand(
                command=[my_python, 'setup.py', 'bdist', '--formats=wininst'],
                env=build_env,
                workdir=build_workdir))
            installer = ['easy_install']
        elif bdist_type == 'whl':
            # Install bdist_wheel command
            do_pip_install(factory, ("wheel>=0.21",), env=build_env)
            # Make wheel - always use virtualenv python
            factory.addStep(ShellCommand(
                command=['python', 'setup.py', 'bdist_wheel'],
                env=build_env,
                workdir=build_workdir))
            installer = ['pip', 'install']
        else:
            raise ValueError('What is this?: ' + bdist_type)
        # Find binary installer file
        factory.addStep(SetPropertyFromCommand(
            command=['python', 'bs_tools.py',
                     'glob', 'dist', '*.' + bdist_type],
            property='bdist_file'))
        # Install into virtualenv
        factory.addStep(ShellCommand(
            command = installer + [Interpolate('%(prop:bdist_file)s')],
            env=env))
        # Test
        kwargs = dict(env=env,
                workdir=workdir,
                haltOnFailure=True)
        if self.build_timeout is not None:
            kwargs['timeout'] = self.build_timeout
        factory.addStep(
            ShellCommand(
                command=test_cmd,
                **kwargs))
        # Upload
        factory.addStep(DirectoryUpload(slavesrc="dist",
                masterdest="public_html/" + self.download_dir))
        return factory

    def doc_factory(self,
                    doc_build_cmd=None,
                    built_doc_src=None,
                    doc_doctest_cmd=None):
        if doc_build_cmd is None:
            doc_build_cmd = self.doc_build_cmd
        if built_doc_src is None:
            built_doc_src = self.built_doc_src
        if None in (doc_build_cmd, built_doc_src):
            raise ValueError("Need defined build command and doc source")
        factory, my_env, workdir, build_workdir = self._code_build_factory()
        # Install into virtualenv
        kwargs = {}
        if self.build_timeout:
            kwargs['timeout'] = self.build_timeout
        factory.addStep(ShellCommand(
            command=['python', 'setup.py', 'install'],
            env=my_env,
            workdir=build_workdir,
            **kwargs))
        # Add install and runtime dependencies
        do_pip_install(factory, self.doc_pip_depends, my_env)
        if not doc_doctest_cmd is None:
            factory.addStep(ShellCommand(command=doc_doctest_cmd, env=my_env))
        factory.addStep(ShellCommand(command=doc_build_cmd, env=my_env))
        factory.addStep(DirectoryUpload(slavesrc=built_doc_src,
                    masterdest="public_html/nightly/" + self.repo + '/doc'))
        return factory

    def build_builders(self, name_slavenames,
                       test_cmd=None,
                       python=None,
                       pip_depends=None,
                       pre_build_cmds=(),
                       post_cmds=(),
                       install_cmd=('python', 'setup.py', 'install'),
                       install_wd=None,
                       branches=None,
                       schedulers=None,   # get a new scheduler assigned per each branch
                       ):
        if branches is None:
            branches = self.branches
        builders = []
        for branch in branches:
            branch_builders = self._factory_builders(
                self.factory(test_cmd,
                             python,
                             pip_depends,
                             pre_build_cmds=pre_build_cmds,
                             post_cmds=post_cmds,
                             install_cmd=install_cmd,
                             install_wd=install_wd,
                             branch=branch),
                name_slavenames,
                None if is_one_default_branch(branches, branch) else branch)

            # assign those to a new unique scheduler if that was requested
            if schedulers is not None:
                schedulers.append(
                    self._get_scheduler([b.name for b in branch_builders],
                                        branches, branch))
            builders += branch_builders
        return builders

    def _factory_builders(self, factory, name_slavenames, branch=None):
        builders = []
        for name, slavelist in name_slavenames:
            if isinstance(slavelist, basestring):
                slavelist = [slavelist]
            if branch is not None:
                name=get_branch_builder_name(name, [None, None], branch)
            builders.append(BuilderConfig(
                name=name,
                slavenames=slavelist,
                factory=factory))
        return builders

    def build_doc_builders(self,
            name_slavenames,
            doc_build_cmd=None,
            built_doc_src=None,
            doc_doctest_cmd=None):
        factory = self.doc_factory(doc_build_cmd, built_doc_src, doc_doctest_cmd)
        return self._factory_builders(factory, name_slavenames)


def nptest_command(pkg_name,
                   verbose=10,
                   doctests=False,
                   python='python',
                   cd=False,
                   stdout=False,
                   exe=False,
                   use_agg=False):
    """ Run tests from new directory """
    pystr = ''
    if cd:
        pystr += "import os; os.chdir('..');"
    if use_agg:
        pystr += 'import os; open("matplotlibrc", "wt").write("backend : agg"); '
    pystr += ("import sys; import %s; sys.exit(not %s.test(doctests=%s"
              % (pkg_name, pkg_name, doctests))
    if verbose != None:
        pystr += ', verbose=%d' % verbose
    if exe:
        pystr += ', extra_argv=["--exe"]'
    pystr += ").wasSuccessful())"
    return [python, '-c', "%s" % pystr]


nibabel_bot = GithubBot('nipy', 'nibabel',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest nibabel nisext'),
    doc_build_cmd = "cd doc && make html",
    doc_pip_depends = ('sphinx', 'texext'),
    built_doc_src = "build/html",
    pip_depends = ('nose', 'mock')
)
nitime_bot = GithubBot('nipy', 'nitime',
    test_cmd = nptest_command('nitime', verbose=None),
    pip_depends = ('nose', 'matplotlib')
)
pyarbus_bot = GithubBot('ivanov', 'pyarbus',
    test_cmd = nptest_command('pyarbus', verbose=None, use_agg=True),
    pip_depends = ('nose','nitime')
)
nipy_bot = GithubBot('nipy', 'nipy',
    test_cmd = nptest_command('nipy', doctests=True),
    doc_build_cmd = "cd doc && make dist",
    built_doc_src = "doc/dist",
    pip_depends = ('nose', 'sympy', 'nibabel', 'numpy', 'scipy'),
    build_timeout = 2400,
)
dipy_bot = GithubBot('nipy', 'dipy',
    test_cmd = ['python', '-c',
    'import nose; '
    'from nose.plugins import doctests; '
    'open("matplotlibrc", "wt").write("backend : agg"); '
    'argv = ["", "dipy", "--with-doctest", "--verbosity=3", "--exe"]; '
    'nose.core.TestProgram(argv=argv, addplugins=[doctests.Doctest()])'],
    pip_depends = ('nose', 'numpy>=1.7.1', 'cython>=0.25.1', 'nibabel', 'scipy'),
    build_timeout = 9600, # allow for longer registration builds
)
# dipy bot for linux machines with optional pip_depends that
# need an X window for testing
dipy_bot_nox = GithubBot('nipy', 'dipy',
    test_env = dict(TEST_WITH_XVFB='skip'),
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --with-doctest dipy'),
    pip_depends = ('nose', 'numpy>=1.7.1', 'cython>=0.18', 'nibabel',
                   'xvfbwrapper'),
    build_timeout = 9600, # allow for longer registration builds
)
nipype_bot = GithubBot('nipy', 'nipype',
    test_cmd = nptest_command('nipype', doctests=True),
    pip_depends = ('Traits', 'nose', 'nibabel', 'networkx', 'python-dateutil')
)
regreg_bot = GithubBot('regreg', 'regreg',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe --with-doctest regreg'),
    pip_depends = ('nose', 'cython'),
)
# pandas bot, primarily for sparc testing
pandas_bot = GithubBot('pydata', 'pandas',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe -s pandas'),
    doc_build_cmd = "make doc",
    built_doc_src = "doc/build/html",
    pip_depends = ('nose', 'apiclient',),
    build_timeout = 7200, # allow for its longer builds/testing especially for sparcs
)
# statsmodels bot, primarily for sparc testing
statsmodels_bot = GithubBot('statsmodels', 'statsmodels',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe -s statsmodels'),
    doc_build_cmd = "make -C docs html",
    built_doc_src = "docs/build",
    pip_depends = ('nose', 'patsy>=0.1.0',),
)
# seaborn bot
seaborn_bot = GithubBot('mwaskom', 'seaborn',
    test_cmd = shlex.split('xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" nosetests --verbosity=3 --exe -s seaborn'),
    pip_depends = ('nose',),
)
# pymvpa bot, primarily for sparc testing
pymvpa_bot = GithubBot('PyMVPA', 'PyMVPA',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe -s mvpa2'),
    test_env = dict(MVPA_TESTS_LABILE='no', MVPA_MATPLOTLIB_BACKEND='agg', MVPA_TESTS_WTF='yes'),
    doc_build_cmd = "make htmldoc",
    built_doc_src = "build/html",
    pip_depends = ('nose', 'statsmodels'),
    build_timeout = 3600, # allow for its longer testing especially for sparcs.
)
# sklearn bot, primarily for sparc testing
sklearn_bot = GithubBot('scikit-learn', 'scikit-learn',
    test_cmd = shlex.split('nosetests --verbosity=3 --exe -s sklearn'),
    doc_build_cmd = "make -C doc html",
    built_doc_src = "doc/build",
    pip_depends = ('nose',),
)
# Fail2Ban bots -- need separate ones for different branches because
# tests are invoked differently across them
fail2ban_bot = GithubBot('fail2ban', 'fail2ban',
    test_cmd = {'master': shlex.split('bin/fail2ban-testcases'),
                '0.8': shlex.split('./fail2ban-testcases')},
    branches = ['master', '0.8'],
    test_dir = './build/',
)

# PsychoPy
psychopy_bot = GithubBot('psychopy', 'psychopy',
    test_cmd = shlex.split('xvfb-run -s "-screen 0 1024x768x24 -ac +extension GLX +render -noreset" python ../psychopy/tests/runPytest.py -v -s -m "not needs_sound"'),
    test_env = dict(USER="nobody", TRAVIS="true"),
    branches = ['master'],
)

# Sympy
sympy_bot = GithubBot('sympy', 'sympy',
    test_cmd = ['python', '-c',
    'import sys; '
    'import sympy; '
    'open("matplotlibrc", "wt").write("backend : agg"); '
    'dt=sympy.doctest(); '
    't=sympy.test(); '
    'st=sympy.test(slow=True, verbose=True, timeout=2300); '
    'sys.exit(not (dt and t and st))'],
    doc_build_cmd = "make html-errors",
    pip_depends = (
        'gmpy==1.16',
        ('--no-deps', 'https://github.com/Theano/Theano/archive/master.zip')),
    doc_pip_depends = ('sphinx==1.1.3',),
    build_timeout = 2400, # allow for very slow sympy tests
)
# Numpy
numpy_bot = GithubBot('numpy', 'numpy',
    test_cmd = nptest_command('numpy', verbose=None),
    pip_depends = ('nose',),
                     )

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

#TODO: We should probably change the GitPoller to a PBChangeSource to react to
# commits http://buildbot.net/buildbot/docs/latest/manual/cfg-changesources.html#chsrc-PBChangeSource
c['change_source'] = [bot.poller() for bot in (nibabel_bot,
                                               nitime_bot,
                                               pyarbus_bot,
                                               nipy_bot,
                                               dipy_bot,
                                               nipype_bot,
                                               regreg_bot,
                                               statsmodels_bot,
                                               seaborn_bot,
                                               pymvpa_bot,
                                               sklearn_bot,
                                               sympy_bot,
                                               numpy_bot,
                                               fail2ban_bot,
                                               psychopy_bot,
                                               )]

# pandas is too expensive to test and overwhelmes the sparc box, so
# let's pull it much less frequently until figuring out how to
# implement **correct** locking
c['change_source'] += [pandas_bot.poller(interval=18000)]


####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes.

c['schedulers'] = sum([
    nibabel_bot.schedulers(['nibabel-py2.6',
                           'nibabel-py2.7-win32',
                           'nibabel-py2.7-win7',
                           'nibabel-py2.6-squeeze-sparc',
                           'nibabel-py2.7-wheezy-sparc',
                           'nibabel-py2.x-sid-sparc',
                           'nibabel-py2.7-ppc',
                           'nibabel-py2.6-arm',
                           'nibabel-py2.7-fedora',
                           'nibabel-py2.7-osx-10.6',
                           'nibabel-py2.7-osx-10.7',
                           'nibabel-py2.7-osx-10.10',
                           'nibabel-py2.7-debian-ppc',
                           'nibabel-py3.4',
                           'nibabel-py3.3',
                           'nibabel-py2.7-pyd0.9.7',
                           'nibabel-py2.7-pyd0.9.8',
                          ]),
    nitime_bot.schedulers(['nitime-py2.6',
                          'nitime-py2.7-win32',
                          'nitime-py2.6-arm',
                          'nitime-py2.7-fedora',
                          'nitime-py2.7-osx-10.6',
                          'nitime-py2.7-osx-10.7',
                          'nitime-py2.7-osx-10.10',
                          'nitime-py3.4',
                          ]),
    pyarbus_bot.schedulers(['pyarbus-py2.6',
                          ]),
    nipy_bot.schedulers(['nipy-py2.6',
                        'nipy-py2.7-win32',
                        'nipy-py2.6-squeeze-sparc',
                        'nipy-py2.7-wheezy-sparc',
                        'nipy-py2.x-sid-sparc',
                        'nipy-py2.7-ppc',
                        'nipy-py2.6-arm',
                        'nipy-py2.7-fedora',
                        'nipy-py2.7-osx-10.6',
                        'nipy-py2.7-osx-10.7',
                        'nipy-py2.7-osx-10.10',
                        'nipy-py2.7-debian-ppc',
                        'nipy-py3.3',
                        'nipy-py3.4',
                        'nipy-py2.7-sym0.7.0-nib1.2.0',
                       ]),
    dipy_bot.schedulers(['dipy-py2.6',
                        'dipy-py2.7-win32',
                        'dipy-py2.6-squeeze-sparc',
                        'dipy-py2.7-wheezy-sparc',
                        'dipy-py2.x-sid-sparc',
                        'dipy-py2.7-fedora',
                        'dipy-py2.7-osx-10.6',
                        'dipy-py2.7-osx-10.7',
                        'dipy-py2.7-osx-10.10',
                        'dipy-py2.7-debian-ppc',
                        'dipy-py3.4',
                        'dipy-py3.3',
                       ]),
    nipype_bot.schedulers(['nipype-py2.6',
                          'nipype-py2.7-win32',
                          'nipype-py2.7-fedora',
                          'nipype-py2.7-osx-10.6',
                          'nipype-py2.7-osx-10.10',
                          'nipype-py2.7-debian-ppc',
                       ]),
    regreg_bot.schedulers(['regreg-py2.6',
                          'regreg-py2.7-win32',
                          'regreg-py2.7-fedora',
                          'regreg-py2.7-osx-10.6',
                          'regreg-py2.7-osx-10.7',
                          'regreg-py2.7-osx-10.10',
                          'regreg-py2.7-debian-ppc',
                       ]),
    pandas_bot.schedulers([
                        'pandas-py2.x-sid-sparc',
                        'pandas-py3.x-sid-sparc',
                        'pandas-py2.6-wheezy-sparc',
                        'pandas-py2.7-wheezy-sparc',
                        'pandas-py3.x-wheezy-sparc',
                        ]),
    statsmodels_bot.schedulers([
                        'statsmodels-py2.x-sid-sparc',
                        # not yet
                        #'statsmodels-py3.x-sid-sparc',
                        'statsmodels-py2.7-wheezy-sparc',
                        #'statsmodels-py3.x-wheezy-sparc',
                        ]),
    sklearn_bot.schedulers([
                        'sklearn-py2.x-sid-sparc',
                        # not yet
                        #'sklearn-py3.x-sid-sparc',
                        'sklearn-py2.6-wheezy-sparc',
                        #'sklearn-py3.x-wheezy-sparc',
                        ]),
    numpy_bot.schedulers([
                        'numpy-py2.7-osx-10.10',
                        'numpy-py3.3-osx-10.10',
                        'numpy-py3.4-osx-10.10',
                        'numpy-py2.7-debian-ppc',
                        'numpy-py2.x-sid-sparc',
                        'numpy-py3.x-sid-sparc',
                        ]),], []) + \
    [
    # Doc builds
    nibabel_bot.nightly_scheduler(['nibabel-doc-builder'], 'doc', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-doc-builder'], 'doc', 2, 01, True),
    pandas_bot.nightly_scheduler(['pandas-doc-builder'], 'doc', 2, 20, True),
    statsmodels_bot.nightly_scheduler(['statsmodels-doc-builder'], 'doc', 3, 20, True),
    pymvpa_bot.nightly_scheduler(['pymvpa-doc-builder'], 'doc', 4, 20, True),
    # Checks for pip build / install
    # Windows 32 nightly builds
    nipy_bot.nightly_scheduler(['nipy-bdist32-27'], 'win32-whl-27', 2, 10, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-33'], 'win32-whl-33', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-34'], 'win32-whl-34', 2, 30, True),
    nipy_bot.nightly_scheduler(['nipy-bdist32-35'], 'win32-whl-35', 2, 30, True),
    # OSX nightlies
    nipy_bot.nightly_scheduler(['nipy-bdist-whl-2.7'], 'osx-whl-27', 2, 20, True),
    nipy_bot.nightly_scheduler(['nipy-bdist-whl-3.3'], 'osx-whl-33', 2, 30, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-27'],
                                  'whl-27', 2, 40, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-33'],
                                  'whl-33', 2, 50, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-34'],
                                  'whl-34', 3, 00, True),
    nibabel_bot.nightly_scheduler(['nibabel-bdist32-35'],
                                  'whl-35', 3, 00, True),
    # Windows 64 nightly builds
    nibabel_bot.nightly_scheduler(['nibabel-bdist64-27'],
                                  'whl64-27', 2, 00, True),
    # Dipy
    # Windows 32 nightly builds
    dipy_bot.nightly_scheduler(['dipy-bdist32-27'], 'dipy-win32-whl-27', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-33'], 'dipy-win32-whl-33', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-34'], 'dipy-win32-whl-34', 3, 30, True),
    dipy_bot.nightly_scheduler(['dipy-bdist32-35'], 'dipy-win32-whl-35', 3, 30, True),
    # Windows 64 nightly
    dipy_bot.nightly_scheduler(['dipy-bdist64-27'], 'dipy-win64-whl-27', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist64-35'], 'dipy-win64-whl-35', 4, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist64-35-win10'], 'dipy-win64-whl-35-win10', 3, 10, True),
    dipy_bot.nightly_scheduler(['dipy-bdist64-35-win2016'], 'dipy-win64-whl-35-win2016', 3, 10, True),
    # OSX nightlies
    dipy_bot.nightly_scheduler(['dipy-bdist-whl-2.7'], 'dipy-osx-whl-27', 3, 20, True),
    dipy_bot.nightly_scheduler(['dipy-bdist-whl-3.3'], 'dipy-osx-whl-33', 3, 30, True),
    # Sympy binary builders
    sympy_bot.nightly_scheduler(['sympy-bdist32-27'],
                                  'whl-27', 5, 30, True),
    sympy_bot.nightly_scheduler(['sympy-bdist64-27'],
                                  'whl64-27', 5, 30, True),
    # Numpy binary builders
    numpy_bot.nightly_scheduler(['numpy-bdist-whl-osx-2.7'], 'whl-27', 5, 30, True),
    numpy_bot.nightly_scheduler(['numpy-bdist-whl-osx-3.5'], 'whl-35', 5, 30, True),
    numpy_bot.nightly_scheduler(['numpy-bdist-whl-osx-3.6'], 'whl-36', 5, 30, True),
]

# Try scheduler defined after builders

# Some factories

def factory_buildout_nibabel(python='python2.6',numpy='1.6.1'):
    f = BuildFactory()
    # check out the source
    # XXX: the mastersrc should make appropriate changes based on numpy version
    f.addStep(Git(repourl='git://github.com/nipy/nibabel.git', mode='full'))
    f.addStep(FileDownload(mastersrc="bootstrap.py",
                                          slavedest="bootstrap.py"))
    f.addStep(FileDownload(mastersrc="nibabel_buildout.cfg",
                                          slavedest="buildout.cfg"))
    f.addStep(ShellCommand(command=[python, "bootstrap.py"]))
    f.addStep(ShellCommand(command=["bin/buildout"]))
    f.addStep(ShellCommand(nptest_command('nibabel',python='bin/buildoutpython')))
    # run the tests (note that this will require that 'nosetests' is installed)
    #f.addStep(nosetests_command('nibabel', doctests=True))
    return f


def shell_cmd_factory(git_url, cmds, posix=True):
    # Run a series of commands over a git repo
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='full', retry=(15, 5)))
    for cmd in cmds:
        factory.addStep(ShellCommand(command=shlex.split(cmd, posix=posix)))
    return factory


def example_factory(git_url, py_path, out_dir, python_cmd='python',
                    pip_depends=(), timeout=2400):
    factory = BuildFactory()
    factory.addStep(Git(
        repourl=git_url,
        mode='full', retry=(15, 5)))
    # Touch c files to make sure build doesn't fail because of c file checkout
    # times
    factory.addStep(ShellCommand(command=
                                 ['python', 'tools/touch_cython_cs.py']))
    # Make virtualenv into which to install
    factory.addStep(
        ShellCommand(command=[
            py_path + r'/bin/virtualenv', '--system-site-packages', 'venv']
            ))
    venv_env = {'PATH': Interpolate("%(prop:workdir)s/build/venv/bin:${PATH}")}
    # Install any dependencies
    for pkg_spec in pip_depends:
        if isinstance(pkg_spec, basestring):
            pkg_spec = [pkg_spec]
        cmd = ['pip', 'install']
        factory.addStep(ShellCommand(command=cmd + list(pkg_spec),
                    env=venv_env))
    # Install into virtualenv
    factory.addStep(
        ShellCommand(command=[python_cmd, 'setup.py', 'install'],
                     env=venv_env)
            )
    # Make directory for examples
    factory.addStep(
        ShellCommand(command=['mkdir', 'eg_logs']))
    # Run examples
    factory.addStep(
        ShellCommand(command=[
            python_cmd,
            '../tools/run_log_examples.py',
            '../examples',
            '--log-path=../eg_logs'],
            workdir=r'build/venv',
            timeout=timeout,
            env=venv_env
            ))
    # Upload logs
    factory.addStep(DirectoryUpload(slavesrc="eg_logs",
            masterdest="public_html/" + out_dir))
    return factory

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

c['builders'] = []
# nibabel exe builders
nibabel_test_cmd = ['nosetests', '--verbosity=3', '--with-doctest', 'nibabel']
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-27',
    slavenames=win7_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'whl',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python27\python.exe')
))
# The Python32 numpy 1.6.2 installation needs patching according to
# git diff ad9c2f4~1..ad9c2f4
# to remove use of __stdout__ and __stderr__ in numpy distutils
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-33',
    slavenames=win7_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'whl',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python33\python.exe')
))
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-34',
    slavenames=win7_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'whl',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python34\python.exe')
))
c['builders'].append(BuilderConfig(
    name='nibabel-bdist32-35',
    slavenames=win7_32_slaves,
    factory=nibabel_bot.bdist_factory(
        'whl',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python35\python.exe')
))
c['builders'].append(BuilderConfig(
    name='nibabel-bdist64-27',
    slavenames=win7_64_slaves,
    factory=nibabel_bot.bdist_factory(
        'whl',
        test_cmd = nibabel_test_cmd,
        python = r'c:\Python27\python.exe')
))
# nipy exe builders
nipy_test_cmd = nptest_command('nipy', doctests=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-27',
    slavenames=win7_32_slaves,
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python27\python.exe',
        pre_build_cmds = [['python', r'tools\touch_cython_cs.py']])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-33',
    slavenames=win7_32_slaves,
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python33\python.exe',
        pre_build_cmds = [['python', r'tools\touch_cython_cs.py']])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-34',
    slavenames=win7_32_slaves,
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python34\python.exe',
        pre_build_cmds = [['python', r'tools\touch_cython_cs.py']])
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist32-35',
    slavenames=win7_32_slaves,
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd = nipy_test_cmd,
        python = r'c:\Python35\python.exe',
        pre_build_cmds = [['python', r'tools\touch_cython_cs.py']])
))
# Add bdist builders for OSX
# Need exe because easy_install makes tests executable on 3.3
nipy_test_cmd = nptest_command('nipy', doctests=True, exe=True)
c['builders'].append(BuilderConfig(
    name='nipy-bdist-whl-2.7',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='nipy-bdist-whl-3.3',
    slavenames=['osx-10.6'],
    factory=nipy_bot.bdist_factory(
        'whl',
        test_cmd=nipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
##############################
# Dipy builders
##############################
# We have to jump through some difficult hoops to put the `dipnost` command on
# the path, because multiprocessing gets very unhappy if you try and run stuff
# from a command in a file that is not importable on the Python path, at least
# on Windows 32 bit
win32_dipy_test_cmd = ['python',  r'dipnost.py', 'dipy', '--verbosity=3']
dipy_pre_build = [['copy', r'tools\dipnost', r'venv\dipnost.py']]
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-27',
    slavenames=win7_32_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python27\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist64-27',
    slavenames=win7_64_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python27\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist64-35',
    slavenames=win7_64_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python35\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist64-35-win10',
    slavenames=win10_64_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python35\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist64-35-win2016',
    slavenames=win2016_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python36-64\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-33',
    slavenames=win7_32_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python33\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-34',
    slavenames=win7_32_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python34\python.exe')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist32-35',
    slavenames=win7_32_slaves,
    factory=dipy_bot.bdist_factory(
        'whl',  # nose appears broken finding stuff in eggs
        # Copy dipnost as py file into (later) working directory
        pre_build_cmds=dipy_pre_build,
        test_cmd=win32_dipy_test_cmd,
        python=r'c:\Python35\python.exe')
))
# Add bdist builders for OSX
# Each of the tested _system_ pythons will need:
# * numpy, scipy, virtualenv
osx_dipy_test_cmd = ['python',  '../tools/dipnost', '--exe', '--verbosity=3', 'dipy']
c['builders'].append(BuilderConfig(
    name='dipy-bdist-whl-2.7',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'whl',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/2.7/bin/python')
))
c['builders'].append(BuilderConfig(
    name='dipy-bdist-whl-3.3',
    slavenames=['osx-10.6'],
    factory=dipy_bot.bdist_factory(
        'whl',
        test_cmd=osx_dipy_test_cmd,
        python='/Library/Frameworks/Python.framework/Versions/3.3/bin/python3')
))
# sympy binary builders
sympy_test_cmd = ['python', '-c',
                  'import sys; '
                  'import sympy; '
                  'open("matplotlibrc", "wt").write("backend : agg"); '
                  'dt=sympy.doctest(); '
                  't=sympy.test(); '
                  'sys.exit(not (dt and t))']
pip_deps = ('nose',)
for slavenames, bdist_suffix in (
    (win7_32_slaves, '32'),
    (win7_64_slaves, '64')):
    py2_name = 'sympy-bdist{0}-27'.format(bdist_suffix)
    py3_sched_name = 'sympy-bdist{0}-33-scheduler'.format(bdist_suffix)
    py3_build_name = 'sympy-bdist{0}-33-builder'.format(bdist_suffix)
    factory = sympy_bot.bdist_factory(
        'whl',
        test_cmd = sympy_test_cmd,
        pip_depends = pip_deps,
        python = r'c:\Python27\python.exe')
    factory.addStep(
            SetPropertyFromCommand(
                command=['python', 'bs_tools.py', 'get-tail',
                         Property('bdist_file')],
                property='bdist_fname'))
    factory.addStep(
        Trigger(schedulerNames=[py3_sched_name],
                updateSourceStamp=True,
                set_properties={'bdist_fname' : Property('bdist_fname')}
            ))
    c['builders'].append(BuilderConfig(
        name=py2_name,
        slavenames=slavenames,
        factory=factory))
    # Make triggerable scheduler
    c['schedulers'].append(
            Triggerable(name=py3_sched_name, builderNames=[py3_build_name]))
    # Make builder for Python 3 bdist test
    factory = BuildFactory()
    env, workdir = make_virtualenv(factory, 'c:\Python33\python.exe',
                                   sys_site_packages=True)
    # Download bdist file
    factory.addStep(
        FileDownload(Interpolate(
            "public_html/" +
            sympy_bot.download_dir +
            "/%(prop:bdist_fname)s"),
            Property('bdist_fname'))
            )
    # pip install bdist file into virtualenv
    factory.addStep(ShellCommand(
        command=['pip', 'install', Property('bdist_fname')],
        env=env))
    # pip install nose
    factory.addStep(ShellCommand(
        command=['pip', 'install', '--ignore-installed', 'nose'],
        env=env))
    # run tests
    factory.addStep(ShellCommand(
        command=sympy_test_cmd,
        env=env,
        workdir=workdir))
    c['builders'].append(BuilderConfig(
        name=py3_build_name,
        slavenames=slavenames,
        factory=factory))
# Make numpy wheels on 10.9, test on 10.6
building_slaves = osx_yosemite_slaves
testing_slaves = osx_snowleopard_slaves
for py_ver in ('2.7', '3.5', '3.6'):
    python_exe = pyorg_pybin(py_ver)
    builder_name = 'numpy-bdist-whl-osx-' + py_ver
    tester_name = builder_name + '-downloaded'
    test_scheduler_name = tester_name + '-scheduler'
    factory = numpy_bot.bdist_factory('whl', python = python_exe)
    factory.addStep(
            SetPropertyFromCommand(
                command=['python', 'bs_tools.py', 'get-tail', Property('bdist_file')],
                property='bdist_fname'))
    factory.addStep(
        Trigger(schedulerNames=[test_scheduler_name],
                updateSourceStamp=True,
                set_properties={'bdist_fname' : Property('bdist_fname')}
            ))
    c['builders'].append(BuilderConfig(
        name=builder_name,
        slavenames=building_slaves,
        factory=factory))
    # Make triggerable scheduler
    c['schedulers'].append(
            Triggerable(name=test_scheduler_name, builderNames=[tester_name]))
    # Make downloading builder for other osx version
    factory = BuildFactory()
    env, workdir = make_virtualenv(factory, python_exe, sys_site_packages=True)
    # Download bdist file
    factory.addStep(
        FileDownload(Interpolate(
            "public_html/" +
            numpy_bot.download_dir +
            "/%(prop:bdist_fname)s"),
            Property('bdist_fname'))
            )
    # Upgrade pip to make sure it can use wheels
    factory.addStep(ShellCommand(
        command=['python', '-m', 'pip', 'install', '-U', 'pip'],
        env=env))
    # pip install bdist file into virtualenv
    factory.addStep(ShellCommand(
        command=['pip', 'install', Property('bdist_fname')],
        env=env))
    # pip install nose
    factory.addStep(ShellCommand(
        command=['pip', 'install', '--ignore-installed', 'nose'],
        env=env))
    # run tests
    factory.addStep(ShellCommand(
        command=numpy_bot.test_cmd,
        env=env,
        workdir=workdir))
    c['builders'].append(BuilderConfig(
        name=tester_name,
        slavenames=testing_slaves,
        factory=factory))

# Add nightly docs builders
c['builders'] += nipy_bot.build_doc_builders(
    (('nipy-doc-builder', linux_64_slaves),),
    doc_doctest_cmd = "cd doc && make doctest"
)
c['builders'] += nibabel_bot.build_doc_builders(
    (('nibabel-doc-builder', linux_64_slaves),))
c['builders'] += pandas_bot.build_doc_builders(
    (('pandas-doc-builder', sparc_slaves_sid),))
c['builders'] += statsmodels_bot.build_doc_builders(
    (('statsmodels-doc-builder', sparc_slaves_sid),))
c['builders'] += pymvpa_bot.build_doc_builders(
    (('pymvpa-doc-builder', sparc_slaves_sid),))

# pre-release builders
def inplace_cmd_factory(bot, commands, **kwargs):
    factory, env, workdir, buildir = bot._code_build_factory(**kwargs)
    factory.addStep(ShellCommand(
        command=('python', 'setup.py', 'build_ext', '-i'),
        env=env))
    for cmd in commands:
        factory.addStep(ShellCommand(command=cmd, env=env))
    return factory

c['builders'].append(BuilderConfig(
    name='nibabel-release-checks',
    slavenames=['osx-10.6'],
    factory=inplace_cmd_factory(nibabel_bot,
        (('python', '-m', 'compileall', '.'),
         ('make', 'sdist-tests'),
         ('make', 'check-files'),
         ('make', 'check-version-info'),
         ('make', 'source-release'),
         ('pip', 'install', '.'),
         ('pip', 'install', '--ignore-installed', 'sphinx'),
         ('pip', 'install', '-r', 'doc-requirements.txt'),
         ('make', '-C', 'doc', 'doctest'),
        ),
        )))
c['builders'].append(BuilderConfig(
    name='nipy-release-checks',
    slavenames=['osx-10.6'],
    factory=inplace_cmd_factory(nipy_bot,
        (('python', '-m', 'compileall', '.'),
         ('make', 'sdist-tests'),
         ('make', 'bdist-egg-tests'),
         ('make', 'check-version-info'),
         ('make', 'check-files')),
                               )))
c['builders'].append(BuilderConfig(
    name='dipy-release-checks',
    slavenames=['osx-10.6'],
    factory=inplace_cmd_factory(dipy_bot,
        (('python', '-m', 'compileall', '.'),
         ('make', 'sdist-tests'),
         ('make', 'bdist-egg-tests'),
         ('make', 'check-version-info'),
         ('make', 'check-files')),
        )))

# Example builders
c['builders'].append(BuilderConfig(
    name='nipy-examples-2.7',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/2.7',
        "nipy-examples-2.7",
        pip_depends=('nibabel', 'sympy', 'matplotlib'),
        timeout=4800,
)
))
c['builders'].append(BuilderConfig(
    name='nipy-examples-3.4',
    slavenames=['osx-10.6'],
    factory=example_factory(
        'git://github.com/nipy/nipy.git',
        '/Library/Frameworks/Python.framework/Versions/3.4',
        "nipy-examples-3.4",
        "python3.4",
        pip_depends=('nibabel', 'sympy', 'matplotlib'),
        timeout=4800,
)
))
# pip pypi builders
hours = 5
minutes = 0
for bot in (nibabel_bot, nipy_bot, dipy_bot):
    for pyver in ('2.7', '3.5'):
        project = bot.repo
        install_cmd = ('pip', 'install', '--ignore-installed', project)
        name_suff =  'py%s-%s' % (pyver, install_cmd[0])
        builder_name = '%s-%s' % (project, name_suff)
        c['builders'] += bot.build_builders(
            ((builder_name, osx_yosemite_slaves),),
            python=pyorg_pybin(pyver),
            pip_depends = ('nose', 'mock'),
            install_cmd = install_cmd,
            install_wd = 'build/venv')
        c['schedulers'] += bot.nightly_scheduler(
                [builder_name],
                name_suff,
                hours,
                minutes,
                False),  # build regardless of git state
        minutes += 10
        if minutes == 60:
            hours += 1
            minutes = 0

# Code builders
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.6', linux_64_slaves),
     ('nibabel-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nibabel-py2.7-osx-10.7', osx_lion_slaves),
     ('nibabel-py2.7-osx-10.10', osx_yosemite_slaves),
     ('nibabel-py2.7-debian-ppc', debian_ppc_slaves),
     ('nibabel-py2.7-ppc', debian_ppc_slaves),
     ('nibabel-py2.6-arm', arm_slaves),
     ('nibabel-py2.7-fedora', fedora_slaves),
     ('nibabel-py2.7-win32', win7_32_slaves),
     ('nibabel-py2.7-win7', win7_64_slaves),
     ('nibabel-py2.6-squeeze-sparc', sparc_slaves),
     ('nibabel-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('nibabel-py2.x-sid-sparc', sparc_slaves_sid)))
# Python 3s into virtualenv
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.4', linux_64_slaves),),
    python='python3.4')
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Different versions of Pydicom
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.7', osx_yosemite_slaves),),
    python='python2.7',
    pip_depends = ('nose', 'mock', 'pydicom==0.9.7'))
c['builders'] += nibabel_bot.build_builders(
    (('nibabel-py2.7-pyd0.9.8', osx_yosemite_slaves),),
    python='python2.7',
    pip_depends = ('nose', 'mock', 'pydicom==0.9.8'))
# Nitime
c['builders'] += nitime_bot.build_builders(
    (('nitime-py2.6', linux_64_slaves),
     ('nitime-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nitime-py2.7-osx-10.7', osx_lion_slaves),
     ('nitime-py2.7-osx-10.10', osx_yosemite_slaves),
     ('nitime-py2.6-arm', arm_slaves),
     ('nitime-py2.7-fedora', fedora_slaves),
     ('nitime-py2.7-win32', win7_32_slaves))
    )
c['builders'] += nitime_bot.build_builders(
    (('nitime-py3.4', linux_64_slaves),),
    python='python3.4')

# pyarbus
c['builders'] += pyarbus_bot.build_builders(
    (('pyarbus-py2.6', linux_64_slaves),))
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.6', linux_64_slaves),
     ('nipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipy-py2.7-osx-10.7', osx_lion_slaves),
     ('nipy-py2.7-osx-10.10', osx_yosemite_slaves),
     ('nipy-py2.7-debian-ppc', debian_ppc_slaves),
     ('nipy-py2.7-ppc', debian_ppc_slaves),
     ('nipy-py2.6-arm', arm_slaves),
     ('nipy-py2.7-fedora', fedora_slaves),
     ('nipy-py2.7-win32', win7_32_slaves),
     ('nipy-py2.6-squeeze-sparc', sparc_slaves),
     ('nipy-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('nipy-py2.x-sid-sparc', sparc_slaves_sid)))
# Python 3s into virtualenv
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
c['builders'] += nipy_bot.build_builders(
    (('nipy-py3.4', linux_64_slaves),),
    python='python3.4')
# Different versions of Sympy, nibabel
c['builders'] += nipy_bot.build_builders(
    (('nipy-py2.7-sym0.7.0-nib1.2.0', osx_yosemite_slaves),),
    pip_depends = ('nose', 'sympy==0.7.0', 'nibabel==1.2.0'))
# The dipy builder seems to fail early sometimes, on OSX, maybe due to:
# http://bugs.python.org/issue8458
c['builders'] += dipy_bot_nox.build_builders(
    (('dipy-py2.6', linux_64_slaves),
     ('dipy-py2.6-squeeze-sparc', sparc_slaves),
     ('dipy-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('dipy-py2.x-sid-sparc', sparc_slaves_sid)
     ))

c['builders'] += dipy_bot.build_builders((
     ('dipy-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('dipy-py2.7-osx-10.7', osx_lion_slaves),
     ('dipy-py2.7-osx-10.10', osx_yosemite_slaves),
     ('dipy-py2.7-debian-ppc', debian_ppc_slaves),
     ('dipy-py2.7-fedora', fedora_slaves),
     ('dipy-py2.7-win32', win7_32_slaves)
     ))
# Python 3.2 into virtualenv
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.4', linux_64_slaves),),
    python='python3.4')
c['builders'] += dipy_bot.build_builders(
    (('dipy-py3.3', osx_snowleopard_slaves),),
    python='python3.3')
# Nipype
c['builders'] += nipype_bot.build_builders(
    (('nipype-py2.6', linux_64_slaves),
     ('nipype-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('nipype-py2.7-osx-10.10', osx_yosemite_slaves),
     ('nipype-py2.7-debian-ppc', debian_ppc_slaves),
     ('nipype-py2.7-fedora', fedora_slaves),
     ('nipype-py2.7-win32', win7_32_slaves)))
c['builders'] += regreg_bot.build_builders(
    (('regreg-py2.6', linux_64_slaves),
     ('regreg-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('regreg-py2.7-osx-10.7', osx_lion_slaves),
     ('regreg-py2.7-osx-10.10', osx_yosemite_slaves),
     ('regreg-py2.7-debian-ppc', debian_ppc_slaves),
     ('regreg-py2.7-fedora', fedora_slaves),
     ('regreg-py2.7-win32', win7_32_slaves)))
# Pandas - can be tested only on recent debian releases
c['builders'] += pandas_bot.build_builders(
    (('pandas-py2.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py2.7-wheezy-sparc', sparc_slaves_wheezy)),
    post_cmds=("python -c 'from pandas.util.print_versions import show_versions; show_versions()'",))
# test on wheezy for python2.6
c['builders'] += pandas_bot.build_builders(
    (('pandas-py2.6-wheezy-sparc', sparc_slaves_wheezy),),
    python='python2.6')
# and python3 builders
c['builders'] += pandas_bot.build_builders(
    (('pandas-py3.x-sid-sparc', sparc_slaves_sid),
     ('pandas-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
    post_cmds=("python3 -c 'from pandas.util.print_versions import show_versions; show_versions()'",),
    python='python3')
# Statsmodels - can be tested only on recent debian releases
c['builders'] += statsmodels_bot.build_builders(
    (('statsmodels-py2.x-sid-sparc', sparc_slaves_sid),
     ('statsmodels-py2.7-wheezy-sparc', sparc_slaves_wheezy)),
    post_cmds=("python -c 'from statsmodels.tools.print_version import show_versions; show_versions()'",))
# Seaborn
c['builders'] += seaborn_bot.build_builders(
    (('seaborn-py2.x-sid-sparc', sparc_slaves_sid),
     ('seaborn-py2.7-wheezy-sparc', sparc_slaves_wheezy)),
    schedulers=c['schedulers']
    )
## and python3 builders
#c['builders'] += statsmodels_bot.build_builders(
#    (('statsmodels-py3.x-sid-sparc', sparc_slaves_sid),
#     ('statsmodels-py3.x-wheezy-sparc', sparc_slaves_wheezy)),
#    python='python3')
# PyMVPA
c['builders'] += pymvpa_bot.build_builders(
    (('pymvpa-py2.x-sid-sparc', sparc_slaves_sid),
     ('pymvpa-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('pymvpa-py2.7-osx-10.6', osx_snowleopard_slaves),
     ('pymvpa-py2.7-osx-10.7', osx_lion_slaves),
     ('pymvpa-py2.7-osx-10.10', osx_yosemite_slaves),
     ('pymvpa-py2.7-debian-ppc', debian_ppc_slaves),
     ('pymvpa-py2.7-win32', win7_32_slaves),
    ), schedulers=c['schedulers'])
# Sklearn
c['builders'] += sklearn_bot.build_builders((
    ('sklearn-py2.x-sid-sparc', sparc_slaves_sid),))
c['builders'] += sklearn_bot.build_builders((
    ('sklearn-py2.6-wheezy-sparc', sparc_slaves_wheezy),),
    python='python2.6')
""" Disabling these guys; they haven't worked for a while
c['builders'].append(
    BuilderConfig(name="buildout-test2.6",
      slavenames=experimental,
      factory=factory_buildout_nibabel()))
c['builders'].append(
    BuilderConfig(name="buildout-test2.5",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.5')))
c['builders'].append(
    BuilderConfig(name="buildout-test2.4",
      slavenames=experimental,
      factory=factory_buildout_nibabel('python2.4')))
"""
# Fail2ban
c['builders'] += fail2ban_bot.build_builders(
    (('fail2ban-py2.x-sid-sparc', sparc_slaves_sid),
     ('fail2ban-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('fail2ban-py2.6-osx-10.6', osx_snowleopard_slaves),
     ('fail2ban-py2.7-osx-10.10', osx_yosemite_slaves),
     ('fail2ban-py2.7-ppc', debian_ppc_slaves),
     ('fail2ban-py2.6-arm', arm_slaves),
     # ('fail2ban-py2.7-win7', win7_64_slaves),
    ),
    schedulers=c['schedulers'],           # in-place modify
    install_cmd=None)
# master does not support 2.5
c['builders'] += fail2ban_bot.build_builders(
    (
        ('fail2ban-py2.7-debian-ppc', debian_ppc_slaves),),
    branches=['0.8'],
    install_cmd=None)
# Psychopy
c['builders'] += psychopy_bot.build_builders(
    (('psychopy-py2.x-sid-sparc', sparc_slaves_sid),
     ('psychopy-py2.7-wheezy-sparc', sparc_slaves_wheezy),
     ('psychopy-py2.6-osx-10.6', osx_snowleopard_slaves),
     ('psychopy-py2.7-osx-10.10', osx_yosemite_slaves),
     ('psychopy-py2.7-ppc', debian_ppc_slaves),
     ('psychopy-py2.6-arm', arm_slaves),
     ('psychopy-py2.7-win7', win7_64_slaves),
    ),
    schedulers=c['schedulers'],           # in-place modify
    install_cmd=None)
# Sympy
c['builders'] += numpy_bot.build_builders(
    (('numpy-py2.7-osx-10.10', osx_yosemite_slaves),
     ('numpy-py3.3-osx-10.10', osx_yosemite_slaves),
     ('numpy-py3.4-osx-10.10', osx_yosemite_slaves),
     ('numpy-py2.7-debian-ppc', debian_ppc_slaves),
     ('numpy-py2.x-sid-sparc', sparc_slaves_sid),
    ))
c['builders'] += numpy_bot.build_builders(
    (('numpy-py3.x-sid-sparc', sparc_slaves_sid),
     ), python='python3')



class PipInstaller(object):
    def __init__(self,
                 pip_installs,
                 python=None,
                 test_commands=(),
                 pip_args=(),
                 find_links=(),
                 build_timeout=1200):
        self.pip_installs = pip_installs
        self.python = python
        self.test_commands = test_commands
        self.pip_args = pip_args
        self.find_links = find_links
        self.build_timeout = build_timeout

    @property
    def full_args(self):
        trusted = ['--trusted-host=' + s[7:] for s in self.find_links
                   if s.startswith('http://')]
        links = ['--find-links=' + s for s in self.find_links]
        return trusted + links + list(self.pip_args)

    def _make_factory(self):
        return BuildFactory()

    def _fill_factory(self, factory):
        # Install into virtualenv
        env, workdir = make_virtualenv(factory, python=self.python)
        add_bs_tools(factory)
        return factory, env, workdir

    def _pip_installs(self, factory, env, workdir):
        do_pip_install(factory, self.pip_installs, env, self.full_args)
        return factory

    def _add_tests(self, factory, env, workdir):
        for test_command in test_commands:
            factory.addStep(
                ShellCommand(command=test_command,
                             env=env,
                             workdir=workdir,
                             timeout=self.build_timeout))
        return factory

    def factory(self):
        factory = self._make_factory()
        factory, env, workdir = self._fill_factory(factory)
        factory = self._pip_installs(factory, env, workdir)
        return self._add_tests(factory, env, workdir)


class CleanInstaller(PipInstaller):
    """ Factory for clean install of Python / pip
    """
    def __init__(self,
                 python_full_version,
                 pip_installs,
                 python=None,
                 test_commands=(),
                 pip_args=(),
                 find_links=(),
                 build_timeout=1200):
        self.python_full_version = python_full_version
        super(CleanInstaller, self).__init__(
            pip_installs, python, test_commands, pip_args, find_links,
            build_timeout)

    def _make_factory(self):
        # Install fresh Python and pip
        factory = BuildFactory()
        factory.addStep(ShellCommand(command=
            ['sudo', '/usr/local/bin/install_python.py',
             self.python_full_version]))
        return factory

    def _pip_installs(self, factory, env, workdir):
        # Install from name, version, pre_tf tuples
        for name, version, pre_tf in self.pip_installs:
            if version:
                if version == '-r':
                    names = ['-r', name]
                elif pre_tf:
                    raise ValueError("cannot use --pre flag with version")
                else:
                    names = ['{0}=={1}'.format(name, version)]
            else:
                names = [name]
            if pre_tf:
                names = ['--pre'] + names
            cmd = ['pip', 'install'] + self.full_args + names
            factory.addStep(
                ShellCommand(command=cmd, env=env, workdir=workdir))
        return factory


def simple_tester(pkg_name, extra_args=''):
    """ A simpler version of nptest_command """
    return ['python', '-c',
            'import sys; '
            'import {pkg_name}; '
            'result = {pkg_name}.test({extra_args}, verbose=3); '
            'sys.exit(not result.wasSuccessful())'.format(
                pkg_name=pkg_name,
                extra_args=extra_args)]


WHEEL_SERVER = HTTP_SERVER_URL
# Cached copy of
# https://raw.githubusercontent.com/matplotlib/matplotlib/master/tests.py
MPL_TESTS_FNAME = 'mpl_tests.py'
MPL_TESTS_URL = HTTP_CACHE_URL + MPL_TESTS_FNAME
SCIPY_EXCLUDE_TESTS = [
    # https://github.com/scipy/scipy/issues/3853
    'test_no_64',
    'test_resiliency_all_32',
    'test_resiliency_all_64',
    'test_resiliency_limit_10',
    'test_resiliency_random',
    'test_ufunc_object_array',
    'test_unary_ufunc_overrides',
    'test_binary_ufunc_overrides',
]
scipy_extra_argv = ['--exclude=' + regex for regex in SCIPY_EXCLUDE_TESTS]

scipy_test_args = 'extra_argv=[{0}]'.format(
    ', '.join(['"{0}"'.format(p) for p in scipy_extra_argv]))

scipy_test_cmd = simple_tester('scipy', scipy_test_args)
mpl_test_setup1 = ['python', Property('bs_tools'), 'get-url', MPL_TESTS_URL, MPL_TESTS_FNAME]
mpl_test_setup2 = ['python', '-c',
                   'import os, shutil; '
                   'shutil.rmtree(os.path.expanduser("~/.matplotlib"))']
# Known mpl test failure for Python 3.4
# https://github.com/matplotlib/matplotlib/pull/2981
mpl_test_cmd = ['python', MPL_TESTS_FNAME]
numexpr_test_cmd = simple_tester('numexpr')
h5_test_cmd = ['python', '-c',
               'import sys; '
               'import unittest; '
               'suite = unittest.TestLoader().discover("h5py"); '
               'res = unittest.TextTestRunner(verbosity=3).run(suite); '
               'sys.exit(not res.wasSuccessful)']
sympy_whl_test_cmd = ['python', '-c',
                      'import sys; '
                      'import sympy; '
                      # too slow for full tests
                      'sys.exit(not sympy.test("/basic", "/util"))']
test_commands = [['nosetests', 'markupsafe'],
                 # Tornado tests failing on this system
                 # See: https://github.com/tornadoweb/tornado/issues/1228
                 # ['python', '-m', 'tornado.test.runtests'],
                 nptest_command('numpy', verbose=None),
                 scipy_test_cmd,
                 mpl_test_setup1,
                 mpl_test_setup2,
                 mpl_test_cmd,
                 ['iptest'],
                 # Some known fails for pandas tests version 0.15
                 ['nosetests', 'pandas',
                 # https://groups.google.com/d/msg/pydata/AzMPFAE9bhw/hs4H516gS4YJ
                 '-e', 'test_clipboard'],
                 h5_test_cmd,
                 numexpr_test_cmd,
                 sympy_whl_test_cmd]

default_requires = WHEEL_SERVER + 'scipy-stats-data-1.0.txt'
wheel_requires = Property('wheel_requires', default_requires)
build_hour = 11
for py_version in '2.7.11', '3.4.3', '3.5.1':
    for find_links, pre, suffix in (((), False, 'pypi'),
                                    ((WHEEL_SERVER,), False, 'staging'),
                                    ((WHEEL_SERVER,), True, 'pre')):
        c['builders'].append(BuilderConfig(
            name='scipy-stack-{0}-wheel-{1}'.format(py_version, suffix),
            slavenames=osx_wheel_slaves,
            factory = CleanInstaller(py_version,
                                     (('nose', '', pre),
                                      ('numpy', '', pre),
                                      ('scipy', '', pre),
                                      ('matplotlib', '', pre),
                                      ('numexpr', '', pre),
                                      ('pandas', '', pre),
                                      ('sympy', '', pre),
                                      ('ipython[notebook,test]', '', pre),
                                      ('h5py', '', pre)),
                                     test_commands = test_commands,
                                     build_timeout=2400,
                                     find_links = find_links).factory()))
    requires_builder_name = 'scipy-stack-{0}-wheel-requires'.format(py_version)
    c['builders'].append(BuilderConfig(
        name=requires_builder_name,
        slavenames=osx_wheel_slaves,
        factory = CleanInstaller(py_version,
                                 ((wheel_requires, '-r', False),),
                                 test_commands = test_commands,
                                 build_timeout=2400,
                                 find_links = ()).factory()))
    requires_scheduler_name = requires_builder_name + "-scheduler"
    build_hour += 1
    c['schedulers'].append(timed.Nightly(
                    name=requires_scheduler_name,
                    branch=None,
                    builderNames=[requires_builder_name],
                    change_filter=None,
                    hour=build_hour,
                    minute=0,
                    onlyIfChanged=False))
    c['schedulers'].append(ForceScheduler(
        name=requires_builder_name,
        builderNames=[requires_builder_name],
        properties= [StringParameter(name="wheel_requires",
                    label="set pip requires file for installation",
                    default=default_requires, size=200)]))


""" Builders / schedulers for manylinux testing
"""
build_hour = 5
find_links = [
    'https://d9a97980b71d47cde94d-aae005c4999d7244ac63632f8b80e089.ssl.cf2.rackcdn.com']
pip_installs = ['nose', 'numpy', 'scipy', 'scikit-learn', 'numexpr', 'pandas']
# Statsmodels breaks with recent pandas, use recent trunk commit
pip_installs += [
    'cython',
    'git+https://github.com/statsmodels/statsmodels@maintenance/0.8.x'
]

test_commands = [
    simple_tester('numpy', '"full"'),
    simple_tester('scipy', '"full"'),
    ['nosetests', 'numexpr'],
    ['nosetests', 'sklearn'],
    ['nosetests', '-e', 'test_vcomp_2', '--verbosity=3', 'statsmodels'],
]

for label, slavenames, py_bin in (
    ('2.7-debian', linux_64_slaves, '/usr/bin/python2.7'),
    ('3.4-debian', linux_64_slaves, '/usr/bin/python3.4'),
    ('3.5-debian', linux_64_slaves, '/usr/bin/python3.5'),
    ('2.7-debian-narrow', linux_64_slaves, '/home/buildslave/.pyenv/versions/2.7.10/bin/python'),
    ('2.7-fedora', fedora_slaves, '/usr/bin/python2.7'),
    ):
    builder_name='manylinux-' + label
    c['builders'].append(BuilderConfig(
        name=builder_name,
        slavenames=slavenames,
        factory = PipInstaller(pip_installs,
                               python=py_bin,
                               test_commands = test_commands,
                               find_links = find_links).factory()))
    c['schedulers'].append(timed.Nightly(
                    name=builder_name + '-scheduler',
                    branch=None,
                    builderNames=[builder_name],
                    change_filter=None,
                    hour=build_hour,
                    minute=0,
                    onlyIfChanged=False))


# Assign all builders categories -- project names, to improve status visualizations
for b in c['builders']:
    b.category = b.name.split('-')[0]

# Add force scheduler to all builders that don't have one
scheduler_names = [s.name for s in c['schedulers']
                   if isinstance(s, ForceScheduler)]
for b in c['builders']:
    if not b.name in scheduler_names:
        c['schedulers'].append(
            ForceScheduler(name=b.name, builderNames=[b.name]))


####### Try scheduler for any build
c['schedulers'].append(
    Try_Jobdir(name="try1",
               builderNames=[builder.name for builder in c['builders']],
               jobdir="jobdir"))

# A build seemed to work with this command from a dipy repo directory
# $ buildbot try -c ssh -u buildbot --vc=git -b dipy-py2.7-osx-10.10 --host=nipy.bic.berkeley.edu --jobdir=~buildbot/nibotmi/jobdir --branch=master

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html, mail
from buildbot.status.web import auth, authz

c['status'].append(mail.MailNotifier(fromaddr="buildbot@nipy.bic.berkeley.edu",
                      mode='failing',
                      categories=('sympy',),
                      sendToInterestedUsers=False,
                      extraRecipients=['asmeurer@gmail.com']))

c['status'].append(mail.MailNotifier(fromaddr="buildbot@nipy.bic.berkeley.edu",
                      mode='failing',
                      categories=('seaborn',),
                      sendToInterestedUsers=False,
                      extraRecipients=['site-buildbot-nipy@onerussian.com', 'mwaskom@stanford.edu']))

c['status'].append(mail.MailNotifier(fromaddr="buildbot@nipy.bic.berkeley.edu",
                      mode='failing',
                      categories=('pymvpa',),
                      sendToInterestedUsers=True,
                      extraRecipients=['site-buildbot-nipy@onerussian.com']))

c['status'].append(mail.MailNotifier(fromaddr="buildbot@nipy.bic.berkeley.edu",
                      mode='failing',
                      categories=('fail2ban',),
                      sendToInterestedUsers=False,
                      extraRecipients=['fail2ban-buildbots@lists.sourceforge.net']))

# Allow builds to be triggered via the web forms
sympy_release_filter = ChangeFilter(category="release", repository="sympy")
rel_sched = SingleBranchScheduler(name="sympy-release",
                                  change_filter=sympy_release_filter,
                                  builderNames=['sympy-bdist32-27',
                                                'sympy-bdist64-27']
                                 )
c['schedulers'].append(rel_sched)

http_auth = auth.HTPasswdAuth('bot_htpasswd')

authz_cfg=authz.Authz(
    auth = http_auth,
    # change any of these to True to enable; 'auth' to require authorization'
    # see the manual for more options
    gracefulShutdown = False,
    forceBuild = 'auth',
    forceAllBuilds = 'auth',
    pingBuilder = False,
    stopBuild = 'auth',
    stopAllBuilds = False,
    cancelPendingBuild = 'auth',
)
c['status'].append(html.WebStatus(
    http_port=8010,
    authz=authz_cfg,
    change_hook_dialects={'base': True}
))

####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "NiPy"
c['titleURL'] = "http://nipy.org"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = local_setup.master_url

####### DB URL

# This specifies what database buildbot uses to store change and scheduler
# state.  You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"

# Set mime types for server serving non buildbot files
from twisted.web.static import File
webdir = File("public_html")
webdir.contentTypes['.egg'] = 'application/octet-stream'
webdir.contentTypes['.whl'] = 'application/octet-stream'

# Allow patches of up to 1M in size
from twisted.protocols.basic import NetstringReceiver
NetstringReceiver.MAX_LENGTH = 1000000

def _select_builders(builders, regex):
    r = re.compile(regex)
    return [b for b in builders
            if r.match(b.name)]


def _test_easy_and_no_easy(c, regex, command, expect=True):
    for builder in _select_builders(c['builders'], regex):
        # 2nd one is virtualenv always
        assert(builder.factory.steps[1].kwargs['command'][0] == 'virtualenv')
        for step in builder.factory.steps:
            if command in step.kwargs.get('command', []):
                if expect:
                    return
                else:
                    raise AssertionError(
                        "%s builders should not invoke %s."
                        " Got builder %s running command %s"
                        % (regex, command, builder.name,
                            ' '.join(step.kwargs.get('command'))))
    if expect:
        raise AssertionError(
            "None of the %s builders invoked expected %s" % (regex, command))


def test_easy_and_no_easy(c):
    _test_easy_and_no_easy(c, '^fail2ban-.*', expect=False, command='easy_install')
    _test_easy_and_no_easy(c, '^nipy-.*', expect=True, command='pip')


def test_fail2ban_08_state(c):
    builders = _select_builders(c['builders'], 'fail2ban-py2\.7-debian-ppc.*')
    # we should still have branch suffix since it is not master
    assert(builders[0].name == 'fail2ban-py2.7-debian-ppc_0.8')


def test_regression(c):
    print "# builders:   ", len(c['builders'])
    print "# schedulers: ", len(c['schedulers'])
    assert(len(c['builders']) == 174)
    assert(len(c['schedulers']) == 242)
    # print [sc.name for sc in c['schedulers']]


test_easy_and_no_easy(c)
test_fail2ban_08_state(c)
test_regression(c)