build

#!/usr/bin/env python

# Copyright (c) 2008-2019 the MRtrix3 contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Covered Software is provided under this License on an "as is"
# basis, without warranty of any kind, either expressed, implied, or
# statutory, including, without limitation, warranties that the
# Covered Software is free of defects, merchantable, fit for a
# particular purpose or non-infringing.
# See the Mozilla Public License v. 2.0 for more details.
#
# For more details, see http://www.mrtrix.org/.

# pylint: disable=redefined-outer-name,invalid-name

usage_string = '''
USAGE

    ./build [-verbose] [-showdep[=target|all]] [target ...]

DESCRIPTION

    This script will compile and link the MRtrix3 source tree. It relies on the
    configuration file produced by ./configure - please ensure you have run
    this first.

    In most cases, a simple invocation is all that is required:

      $ ./build

    If no targets are provided, the command will default to building all
    applications by scanning through the cmd/ folder.

    The target executables will be located in the bin/ folder, and the shared
    library (if requested - the default) will be located in the lib/ folder (or
    in bin/ on Windows). All intermediate temporary files will be located
    within the tmp/ folder.

SPECIAL TARGETS

    clean
       used to remove all compiler-generated files, including objects,
       executables, and shared libraries.

    bash
       used to update the bash completion script. Note that automatic updating
       of this script can be enabled at the configure stage, by running
       './configure -dev' prior to invoking './build'.

    doc
       used to update the command documentation, so that any modifications to
       the inline documentation in commands can be propagated through to the
       user documentation site.

    select name
       used to switch between configs / builds. This stores the current config
       and all compiler-generated files in a folder (called "build.oldname"),
       and restores the config in "build.name". If the named config does not
       already exist, an empty one is created. If "name" is not given the
       currently active config name is reported.

PARALLELISED BUILD

    By default, 'build' will use all available cores to run a parallel build.
    In some instances, this can cause problems (notably out of RAM errors). You
    can control how many jobs 'build' will run concurrently using the
    NUMBER_OF_PROCESSORS environment variable. For example:

      $ NUMBER_OF_PROCESSORS=1 ./build

OPTIONS

    -verbose
       print each command as it is being invoked

    -nowarnings
       do not print out non-fatal compiler messages (warnings, etc)

    -dryrun
       do not actually execute the compiler and linking commands (used for testing)

    -showdep[=target|all]
       print the list of dependencies for every target (with -showdep=target;
       the default) or for all files

    -tree
       [only used with -showdep] print full dependency tree for each file

    -persistent
       keep trying to build regardless of failures, until none of the remaining
       jobs succeed.

    -timings
       write compile times out to timings.log file.

    -nopaginate
       do not feed error log to the paginator, even if running in a TTY

'''



############################################################################
#                          COMMON DEFINITIONS                              #
############################################################################

import atexit, codecs, copy, glob, os, platform, re, shutil, subprocess, sys, tempfile, time, threading
from timeit import default_timer as timer

# on Windows, need to use MSYS2 version of python - not MinGW version:
if sys.executable[0].isalpha() and sys.executable[1] == ':':
  python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python' ]).decode(errors='ignore').splitlines()[0].strip()
  sys.exit (subprocess.call ([ python_cmd ] + sys.argv))


bin_dir = 'bin'
cmd_dir = 'cmd'
lib_dir = 'core'
misc_dir = 'src'
script_dir = os.path.join('lib', 'mrtrix3')
tmp_dir = 'tmp'

cpp_suffix = '.cpp'
h_suffix = '.h'
libname = 'mrtrix'

include_paths = [ misc_dir ]
config_file = None


system = None
dependencies = 0
dep_recursive = False
verbose = False
dryrun = False
persistent = False
nowarnings = False
paginate = True
targets = []


todo, headers, object_deps, file_flags = {}, {}, {}, {}
lock = threading.Lock()
print_lock = threading.Lock()
stop = False
error_stream = None
main_cindex = 0

logfile = open ('build.log', 'wb')
timingfile = None


bcolors = {
    "candidate" : '\033[94m',
    "no known conversion" : '\033[94m',
    "expected" : '\033[93m',
    "^" : '\033[91m',
    "static assertion" : '\033[91m',
    "Linking" : '\033[01;32m',
    "In function" : '\033[01;32m',
    "WARNING" : '\033[95m',
    "Warning" : '\033[95m',
    "warning" : '\033[95m',
    "required from" : '\033[94m',
    "In instantiation of" : '\033[01;32m',
    "In member" : '\033[01;32m',
    "ERROR" : '\033[01;95m',
    "error" : '\033[01;31m',
    "failed" : '\033[91m',
    "note" : '\033[94m'}







def colorize(s):
  out = ''
  for l in s.splitlines():
    for st in bcolors:
      if st in l:
        l = l.replace (st, bcolors[st] + st) + '\033[0m'
        break
    out += l + '\n'
  return out




def pipe_errors_to_less_handler():
  global error_stream
  if error_stream:
    [ fid, name ] = tempfile.mkstemp()
    fid = os.fdopen (fid, 'wb')
    fid.write (colorize(error_stream).encode (errors='ignore'))
    fid.close()
    os.system ("less -RX " + name)
    os.unlink (name)






def disp (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  sys.stdout.write (msg)
  sys.stdout.flush()
  print_lock.release()


def log (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if verbose:
    sys.stdout.write (msg)
    sys.stdout.flush()
  print_lock.release()



def logtime (msg):
  print_lock.acquire()
  timingfile.write (msg.encode (errors='ignore'))
  timingfile.flush()
  print_lock.release()



def error (msg):
  global error_stream
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if error_stream is not None:
    error_stream += msg
  else:
    sys.stdout.write (msg)
    sys.stdout.flush()
  print_lock.release()


def fail (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  sys.stdout.write (msg)
  sys.stdout.flush()
  print_lock.release()
  sys.exit (1)



def split_path (path):
  return path.replace ('\\', '/').split ('/')


def get_real_name (path):
  if os.path.islink (path):
    return os.readlink (path)
  return path


def _get_expected_bin_filenames (exe_suffix, cmd_directory=cmd_dir):
  binary_apps = { 'mrtrix3.pyc' }
  if exe_suffix:
    binary_apps.add('mrtrix.dll')
  for entry in os.listdir(cmd_directory):
    if entry.endswith(cpp_suffix):
      binary_apps.add (entry[:-len(cpp_suffix)] + exe_suffix)
  return binary_apps


def list_expected_bin_files (exe_suffix, cmd_directory=cmd_dir, bin_directory=bin_dir):
  bin_files = []
  binary_apps = _get_expected_bin_filenames(exe_suffix, cmd_directory=cmd_directory)
  for app in sorted(binary_apps):
    if os.path.isfile(os.path.join(bin_directory, app)):
      bin_files.append(os.path.join(bin_directory, app))
  return bin_files


def list_unexpected_bin_files (exe_suffix, cmd_directory=cmd_dir, bin_directory=bin_dir):
  unexpected = []
  if os.path.isdir(bin_directory):
    binary_apps = _get_expected_bin_filenames(exe_suffix, cmd_directory=cmd_directory)
    for filename in sorted(os.listdir(bin_directory)):
      filepath = os.path.normpath(os.path.join(bin_directory, filename))
      if os.path.isfile(filepath) and filename not in binary_apps and not is_likely_text(filepath):
        unexpected.append(filepath)
  return unexpected


def list_untracked_bin_files (directory = '.'):
  process = subprocess.Popen ([ 'git', '-C', directory, 'ls-files', '-x', '.*', '-o', bin_dir ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  filelist = process.communicate()[0]
  if process.returncode == 0:
    return filelist.decode(errors='ignore').splitlines()
  return []


def is_likely_text(filename, nbytes=1024):
  """ detects if file has a shebang or is utf-8 encoded, empty file is deemed non-text
  :arg filename
  :arg nbytes sample size to analyse
  """
  chunk = b''
  try:
    with open(filename, 'rb') as f:
      chunk = f.read(nbytes)
  except IOError as e:
    log('could not read file ' + filename + '\n' + str(e) + '\n')

  if chunk:
    if chunk.startswith(b'\x23\x21'):  # starts with shebang '#!'
      log('is_likely_text: ' + filename + ': starts with shebang \n')
      return True

    try:
      chunk.decode('utf-8')
      log('is_likely_text: ' + filename + 'utf-8 encoded \n')
      return True
    except UnicodeDecodeError:
      pass
  return False



def modify_path (name, tmp=False, strip=None, add=None):
  if strip is not None:
    name = name[:-len(strip)]
  if add is not None:
    name = name + add
  for project_dir in mrtrix_dir:
    relname = os.path.normpath (os.path.relpath (name, project_dir))
    if relname.startswith ('.'):
      continue
    if tmp:
      relname = os.path.join (tmp_dir, relname)
    elif not os.path.relpath (relname, tmp_dir).startswith ('.'):
      relname = os.sep.join (split_path (relname)[1:])
    name = os.path.normpath (os.path.join (project_dir, relname))
    return name





############################################################################
#                          COMMAND-LINE PARSING                            #
############################################################################

command_doc = False
bash_completion = False

for arg in sys.argv[1:]:
  if '-help'.startswith(arg):
    sys.stdout.write (usage_string)
    sys.exit (0)
  elif '-verbose'.startswith(arg):
    verbose = True
  elif '-dryrun'.startswith(arg):
    dryrun = True
  elif '-persistent'.startswith(arg):
    persistent = True
  elif '-nowarnings'.startswith(arg):
    nowarnings = True
  elif '-showdep'.startswith(arg):
    dependencies = 1
  elif arg.startswith ('-showdep='):
    if arg[9:] == 'target':
      dependencies = 1
    elif arg[9:] == 'all':
      dependencies = 2
    else:
      fail ('invalid specified for option "-showdep" (expected target, all)')
  elif '-tree'.startswith(arg):
    dep_recursive = True
  elif '-timings'.startswith(arg):
    timingfile = open ('timings.log', 'wb')
  elif '-nopaginate'.startswith(arg):
    paginate = False
  elif arg[0] == '-':
    fail ('unknown command-line option "' + arg + '"')
  elif arg == 'bash':
    bash_completion = True
  elif arg == 'doc':
    command_doc = True
  elif arg == 'clean':
    targets = [ 'clean' ]
  else:
    targets.append(arg)




if paginate and sys.stdout.isatty():
  error_stream = ''
  atexit.register (pipe_errors_to_less_handler)




############################################################################
#                            PROJECT DETECTION                             #
############################################################################

mrtrix_dir = [ '.' ]
build_script = sys.argv[0]
separate_project = False

while os.path.abspath (os.path.dirname (get_real_name (build_script))) != os.path.abspath (mrtrix_dir[-1]):
  if not separate_project:
    log ('compiling separate project against:' + os.linesep)
  separate_project = True
  build_script = os.path.normpath (os.path.join (mrtrix_dir[-1], get_real_name (build_script)))
  project_dir = os.path.dirname (build_script)
  mrtrix_dir += [ project_dir ]
  include_paths += [ os.path.join (project_dir, misc_dir) ]
  log ('    ' + project_dir + os.linesep + os.linesep)





############################################################################
#                             CONFIG HANDLING                              #
############################################################################

class ConfigException (Exception):
  pass

def get_active_build (directory):
  active_configs = glob.glob (os.path.join (directory, 'build.*.active'))
  if len (active_configs) > 1:
    fail ('ERROR: more than one config is currently marked as active!')
  if active_configs:
    name = active_configs[0]
    if not os.path.isdir (name):
      raise ConfigException ('ERROR: active config (' + name + ') is not a directory')
    if os.listdir (name):
      raise ConfigException ('ERROR: active config directory (' + name + ') is not empty')
    return name[:-len('.active')]
  os.mkdir (os.path.join (directory, 'build.default.active'))
  return os.path.join (directory, 'build.default')



def store_current_build (directory = '.'):
  stored_config = get_active_build (directory)
  os.rename (stored_config + '.active', stored_config)
  disp ('in "' + directory + '": storing "' + stored_config + '"...\n')
  for f in [ tmp_dir, 'config' ] + list_untracked_bin_files (directory) + glob.glob ('lib/libmrtrix*'):
    if os.path.exists (os.path.join (directory, f)):
      os.renames (os.path.join (directory, f), os.path.join (stored_config, f))



def restore_build (config_name, directory = '.'):
  stored_path = os.path.join (directory, 'build.' + config_name)
  active_path = stored_path + '.active'
  if os.path.isdir (stored_path):
    disp ('in "' + directory + '": restoring "' + stored_path + '"...\n')
    os.rename (stored_path, active_path)
    for root, dirs, files in os.walk(active_path, topdown=False):
      for name in files:
        os.renames (os.path.join (root, name), os.path.join (directory, os.path.relpath (root, active_path), name))
      for name in dirs:
        if os.path.isdir (os.path.join (root, name)):
          os.rmdir (os.path.join(root, name))
  else:
    if os.path.exists (stored_path):
      raise ConfigException ('ERROR config to be restored (' + stored_path + ') is not a directory')
    disp ('in "' + directory + '": creating empty "' + stored_path + '"...\n')
  if not os.path.isdir (active_path):
    os.mkdir (active_path)



def activate_build (name, directories):
  for directory in directories:
    config = get_active_build (directory)
    if os.path.realpath (config) == os.path.realpath (os.path.join (directory, 'build.' + name)):
      continue

    store_current_build (directory)
    restore_build (name, directory)






if targets and targets[0] == 'select':

  if len(targets) == 1:
    active_config = os.path.basename (get_active_build ('.'))
    disp ('current config is "' + active_config + '"\n')
    for entry in mrtrix_dir[1:]:
      other_config = os.path.basename (get_active_build (entry))
      if active_config != other_config:
        disp ('WARNING: directory "' + entry + '" contains config "' + other_config + '"\n')
    sys.exit (0)

  if len(targets) != 2:
    fail ('ERROR: select target expects a single configuration name\n')

  activate_build (targets[1], mrtrix_dir)

  sys.exit (0)







active_config = os.path.basename (get_active_build ('.'))[6:]
log ('active config is ' + active_config + '\n\n')
active_config_core = os.path.basename (get_active_build (mrtrix_dir[-1]))[6:]
if active_config_core != active_config:
  disp ('active config differs from core - switching to core active config\n')
activate_build (active_config_core, mrtrix_dir)



############################################################################
#                          LOAD CONFIGURATION FILE                         #
############################################################################

if config_file is None:
  config_file = os.path.normpath (os.path.join (mrtrix_dir[-1], 'config'))

# prevent pylint from generating undefined variable / non-iterable / membership test warnings
PATH = obj_suffix = exe_suffix = lib_prefix = lib_suffix = None
runpath = ld_enabled = moc = rcc = nogui = None
cpp = cpp_flags = ld = ld_flags = ld_lib = ld_lib_flags = eigen_cflags = qt_cflags = qt_ldflags = [ ]

try:
  log ('reading configuration from "' + config_file + '"...' + os.linesep)
  exec (codecs.open (config_file, mode='r', encoding='utf-8').read()) # pylint: disable=exec-used
except IOError:
  fail ('''no configuration file found!
please run "./configure" prior to invoking this script

''')

# renamed internal string substitutions
if 'LDFLAGS' in ld or 'LDLIB_FLAGS' in ld_lib:
  fail ('''configuration file is out of date!
please run "./configure" prior to invoking this script

''')

if separate_project:
  cpp_flags += [ '-DMRTRIX_PROJECT' ]

environ = os.environ.copy()
environ.update ({ 'PATH': PATH })

target_bin_dir = os.path.join (mrtrix_dir[0], bin_dir)
purged_bin_dir = os.path.join (target_bin_dir, 'purged_files')

system = platform.system().lower()
if system.startswith('mingw') or system.startswith('msys'):
  target_lib_dir = os.path.join (mrtrix_dir[-1], bin_dir)
else:
  target_lib_dir = os.path.join (mrtrix_dir[-1], 'lib')
lib_dir = os.path.join (mrtrix_dir[-1], lib_dir)

if ld_enabled and runpath:
  ld_flags += [ runpath+os.path.relpath (target_lib_dir,target_bin_dir) ]




############################################################################
#                            BUILD CLEAN                                   #
############################################################################

if 'clean' in targets:

  for f in [ tmp_dir, 'dev' ]:
    if not os.path.isdir (f):
      continue
    for root, dirs, files in os.walk(f, topdown=False):
      for entry in files:
        filename = os.path.join(root, entry)
        disp ('delete file: ' + filename + '\n')
        try:
          os.remove (filename)
        except OSError as excp:
          disp ('error deleting file "' + filename + '": ' + os.strerror (excp.errno))
      for entry in dirs:
        dirname = os.path.join(root, entry)
        disp ('delete directory: ' + dirname + '\n')
        try:
          os.rmdir (dirname)
        except OSError as excp:
          disp ('error deleting folder "' + dirname + '": ' + os.strerror (excp.errno))
    disp ('delete directory: ' + f + '\n')
    try:
      os.rmdir (f)
    except OSError as excp:
      disp ('error deleting folder "' + f + '": ' + os.strerror (excp.errno))

  for filename in list_expected_bin_files(exe_suffix):
    disp ('delete file: ' + filename + '\n')
    try:
      os.remove (filename)
    except OSError as excp:
      disp ('error deleting file "' + filename + '": ' + os.strerror (excp.errno))

  for filepath in list_unexpected_bin_files(exe_suffix):
    if not os.path.exists(purged_bin_dir):
      os.makedirs(purged_bin_dir)
    move_to = os.path.join(purged_bin_dir, os.path.split(filepath)[1])
    while os.path.isfile(move_to):
      move_to = move_to + '_'
    disp ('WARNING: moving unexpected file ' + filepath + ' to ' + move_to + '\n')
    try:
      shutil.move(filepath, move_to)
    except OSError as excp:
      disp ('error moving file "' + filepath + '": ' + os.strerror (excp.errno))

  for filename in glob.glob (os.path.join ('lib', 'libmrtrix*')):
    if os.path.isfile (filename):
      disp ('delete file: ' + filename + '\n')
      try:
        os.remove (filename)
      except OSError as excp:
        disp ('error deleting file "' + filename + '": ' + os.strerror (excp.errno))

  sys.exit (0)




############################################################################
#                          GET VERSION INFORMATION                         #
############################################################################

if ld_enabled:
  ld_flags.insert(0, '-l' + libname)
  libname = lib_prefix + libname + lib_suffix


# other settings:
include_paths += [ lib_dir, cmd_dir ]
cpp_flags += [ '-I' + entry for entry in include_paths ]
ld_flags += [ '-L' + target_lib_dir ]

moc_cpp_suffix = '_moc' + cpp_suffix
moc_obj_suffix = '_moc' + obj_suffix




# remove any files that might have been left over from older installations in
# different locations:
if os.path.isdir ('release'):
  disp ('WARNING: removing \'release/\' folder - most likely left over from a previous installation\n')
  shutil.rmtree ('release')

for entry in glob.glob (os.path.normpath (os.path.join (target_lib_dir, '*' + lib_suffix))):
  if os.path.basename (entry) != libname:
    disp ('WARNING: removing "' + entry + '" - most likely left over from a previous installation\n')
    os.remove (entry)

# move unexpected binary files out of the target bin directory:
for filepath in list_unexpected_bin_files(exe_suffix):
  if not os.path.exists(purged_bin_dir):
    os.makedirs(purged_bin_dir)
  move_to = os.path.join(purged_bin_dir, os.path.split(filepath)[1])
  while os.path.isfile(move_to):
    move_to = move_to + '_'
  disp ('WARNING: moving unexpected binary file ' + filepath + ' to ' + move_to + '\n')
  try:
    shutil.move(filepath, move_to)
  except OSError as excp:
    disp ('error moving file "' + filepath + '": ' + os.strerror (excp.errno))


###########################################################################
#                           TO-DO LIST ENTRY                              #
###########################################################################

class TargetException (Exception):
  pass

class Entry(object):
  def __init__ (self, name):
    global todo
    name = os.path.normpath (name)
    if name in todo:
      return
    todo[name] = self


    self.name = name
    self.cmd = []
    self.deps = set()
    self.action = '--'
    self.timestamp = mtime (self.name)
    self.dep_timestamp = self.timestamp
    self.currently_being_processed = False

    if is_executable (self.name):
      self.set_executable()
    elif is_icon (self.name):
      self.set_icon()
    elif is_object (self.name):
      self.set_object()
    elif is_library (self.name):
      self.set_library()
    elif is_moc (self.name):
      self.set_moc()
    elif not os.path.exists (self.name):
      raise TargetException ('unknown target "' + self.name + '"')


    [ Entry(item) for item in self.deps ] # pylint: disable=expression-not-assigned
    dep_timestamp = [ todo[item].timestamp for item in todo if item in self.deps and not is_library(item) ]
    dep_timestamp += [ todo[item].dep_timestamp for item in todo if item in self.deps and not is_library(item) ]
    if dep_timestamp:
      self.dep_timestamp = max(dep_timestamp)



  def execute (self, cindex, formatstr):
    folder = os.path.dirname (self.name)
    try:
      os.makedirs (folder)
    except OSError as excp:
      if not os.path.isdir (folder):
        fail ('ERROR: can''t create target folder "' + folder + '": ' + os.strerror (excp.errno))

    if self.action == 'RCC':
      with codecs.open (self.cmd[1], mode='w', encoding='utf-8') as fd:
        fd.write ('<!DOCTYPE RCC><RCC version="1.0">\n<qresource>\n')
        for entry in self.deps:
          entry = os.path.basename (entry)
          if not entry.startswith ('config'):
            fd.write ('<file>' + entry + '</file>\n')
        fd.write ('</qresource>\n</RCC>\n')
    if self.cmd:
      return execute (formatstr.format (cindex, self.action, self.name), self.cmd)
    return None


  def set_executable (self):
    self.action = 'LB'
    if exe_suffix and self.name.endswith(exe_suffix):
      cc_file = self.name[:-len(exe_suffix)]
    else:
      cc_file = self.name
    cc_file = modify_path (os.path.join (cmd_dir, os.sep.join (split_path(cc_file)[1:])), add=cpp_suffix)
    self.deps = list_cmd_deps(cc_file)
    if separate_project:
      self.deps = self.deps.union ([ os.path.join (tmp_dir, misc_dir, 'project_version' + obj_suffix) ])

    skip = False
    flags = []
    if 'Q' in file_flags[cc_file]:
      flags += qt_ldflags

    if not skip:
      if not ld_enabled:
        self.deps = self.deps.union (list_lib_deps())

      self.cmd = fillin (ld, {
          'LINKFLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_flags ] + flags,
          'OBJECTS': list(self.deps),
          'EXECUTABLE': [ self.name ] })

      try:
        if ld_use_shell:
          self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
      except NameError:
        pass

      if ld_enabled:
        self.deps.add (os.path.normpath (os.path.join (target_lib_dir, libname)))



  def set_object (self):
    self.action = 'CC'
    cc_file = self.name[:-len(obj_suffix)] + cpp_suffix

    flags = copy.copy (eigen_cflags)

    if is_moc (cc_file):
      src_header = modify_path (cc_file, strip=moc_cpp_suffix, add=h_suffix)
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    elif is_icon (cc_file):
      src_header = modify_path (cc_file, strip=cpp_suffix, add=h_suffix)
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    else:
      cc_file = modify_path (cc_file, tmp=False)
      self.deps = self.deps.union (list_headers (cc_file))
    self.deps.add (config_file)

    self.deps.add( cc_file )

    skip = False
    if 'Q' in file_flags[cc_file]:
      flags += qt_cflags

    if not skip:
      self.cmd = fillin (cpp, {
          'CFLAGS': cpp_flags + flags,
          'OBJECT': [ self.name ],
          'SRC': [ cc_file ] })


  def set_moc (self):
    self.action = 'MOC'
    src_file = modify_path (self.name, strip=moc_cpp_suffix, add=h_suffix)
    self.deps = set([ src_file ])
    self.deps = self.deps.union (list_headers (src_file))
    self.deps.add (config_file)
    self.cmd = [ moc ]
    self.cmd += [ src_file, '-o', self.name ]


  def set_library (self):
    if not ld_enabled:
      fail ('ERROR: shared library generation is disabled in this configuration')

    self.action = 'LD'
    self.deps = list_lib_deps()

    self.cmd = fillin (ld_lib, {
        'LINKLIB_FLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_lib_flags ],
        'OBJECTS': self.deps,
        'LIB': [ self.name ] })

    try:
      if ld_use_shell:
        self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
    except NameError:
      pass


  def set_icon (self):
    self.action = 'RCC'
    with codecs.open (modify_path (self.name, strip=cpp_suffix, add=h_suffix), mode='r', encoding='utf-8') as fd:
      for line in fd:
        if line.startswith ('//RCC:'):
          for entry in line[6:].strip().split():
            self.deps = self.deps.union (glob.glob (os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', entry))))
    self.deps.add (config_file)
    qrc_file = os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', os.path.basename (os.path.dirname(self.name)) + '.qrc'))
    self.cmd = [ rcc, qrc_file, '-o', self.name ]


  def need_rebuild (self):
    return self.timestamp == float("inf") or self.timestamp < self.dep_timestamp

  def display (self, indent=''):
    show_rebuild = lambda x: x+' [REBUILD]' if todo[x].need_rebuild() else x
    msg = indent + '[' + self.action + '] ' + show_rebuild (self.name) + ':\n'
    msg += indent + '  timestamp: ' + str(self.timestamp)
    if self.deps:
      msg += ', dep timestamp: ' + str(self.dep_timestamp) + ', diff: ' + str(self.timestamp-self.dep_timestamp)
    msg += '\n'
    if self.cmd:
      msg += indent + '  command:   ' + ' '.join(self.cmd) + '\n'
    if self.deps:
      msg += indent + '  deps:      '
      if dep_recursive:
        disp (msg + '\n')
        for x in self.deps:
          todo[x].display (indent + '    ')
      else:
        disp (msg + (indent+'\n             ').join([ show_rebuild(x) for x in self.deps ]) + '\n')






###########################################################################
#                         FUNCTION DEFINITIONS                            #
###########################################################################


def default_targets():
  if not os.path.isdir (cmd_dir):
    fail ('ERROR: no "cmd" folder - unable to determine default targets' + os.linesep)
  for entry in os.listdir (cmd_dir):
    if entry.endswith(cpp_suffix):
      targets.append (os.path.normpath (os.path.join (target_bin_dir, entry[:-len(cpp_suffix)] + exe_suffix)))
  return targets

def is_executable (target):
  return os.path.normpath (split_path (target)[0]) == os.path.normpath (bin_dir) and not is_moc (target) and not is_library (target)

def is_library (target):
  return target.endswith (lib_suffix) and split_path(target)[-1].startswith (lib_prefix)

def is_object (target):
  return target.endswith (obj_suffix)

def is_moc (target):
  return target.endswith (moc_cpp_suffix)

def is_icon (target):
  return os.path.basename (target) == 'icons'+cpp_suffix

def mtime (target):
  if not os.path.exists (target):
    return float('inf')
  return os.stat(target).st_mtime


def fillin (template, keyvalue):
  cmd = []
  for item in template:
    if item in keyvalue:
      cmd += keyvalue[item]
    else:
      cmd += [ item ]
  return cmd




def execute (message, cmd, working_dir=None):
  disp (message + os.linesep)
  log (' '.join(cmd) + os.linesep)

  if dryrun:
    return 0

  try:
    start = timer()
    process = subprocess.Popen (cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environ, cwd=working_dir)
    ( stdout, stderr ) = process.communicate()
    end = timer()
    if timingfile is not None:
      logtime ('[{:>8.3f}s] '.format(end-start) + message + os.linesep)

    if process.returncode != 0:
      error ('\nERROR: ' + message + '\n\n' + ' '.join(cmd) + '\n\nfailed with output\n\n' + stderr.decode (errors='ignore'))
      return 1

    errstr = None
    if stdout or stderr:
      errstr = '\n' + ' '.join(cmd) + ':\n\n'

    if stdout:
      errstr += stdout.decode (errors='ignore') + '\n\n'
    if stderr:
      errstr += stderr.decode (errors='ignore') + '\n\n'

    if errstr and not nowarnings:
      disp (errstr)

  except OSError:
    disp (cmd[0] + ': command not found')
    return 1
  return None


def print_deps (current_file, indent=''):
  current_file = os.path.normpath (current_file)
  msg = indent + current_file
  if current_file in file_flags:
    if file_flags[current_file]:
      msg += ' [' + file_flags[current_file] + ']'
  msg += os.linesep
  disp (msg)
  for entry in todo[current_file].deps:
    print_deps (entry, indent + '    ')




def is_GUI_target (current_file):
  if 'gui' in split_path (current_file):
    return True
  if current_file in file_flags:
    if 'Q' in file_flags[current_file]:
      return True
  if todo[current_file].deps:
    for entry in todo[current_file].deps:
      if is_GUI_target (entry):
        return True
  return False



def list_headers (current_file):
  global headers, file_flags
  current_file = os.path.normpath (current_file)

  if current_file not in headers.keys():
    headers[current_file] = set()

    if current_file not in file_flags:
      file_flags[current_file] = ''

    if 'gui' in split_path (current_file):
      if 'Q' not in file_flags[current_file]:
        file_flags[current_file] += 'Q'
    if not os.path.exists (current_file):
      if os.path.basename(current_file) == 'icons'+cpp_suffix:
        return headers[current_file]
      fail ('ERROR: cannot find file "' + current_file + '"' + os.linesep)
    with codecs.open (current_file, mode='r', encoding='utf-8') as fd:
      for line in fd:
        line = line.strip()
        if line.startswith('#include'):
          line = line[8:].split ('//')[0].split ('/*')[0].strip()
          if line[0] == '"':
            line = line[1:].rstrip('"')
            for path in include_paths:
              if os.path.exists (os.path.join (path, line)):
                line = os.path.normpath (os.path.join (path, line))
                headers[current_file].add (line)
                for entry in list_headers(line):
                  headers[current_file].add (entry)
                break
            else:
              fail ('ERROR: cannot find header file \"' + line + '\" (from file \"' + current_file + '\")' + os.linesep)
        elif line == 'Q_OBJECT':
          if 'M' not in file_flags[current_file]:
            file_flags[current_file] += 'M'

    for entry in headers[current_file]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[current_file]:
          file_flags[current_file] += c

  return headers[current_file]






def list_cmd_deps (file_cc):
  global object_deps, file_flags
  file_cc = os.path.normpath (file_cc)

  if file_cc not in object_deps.keys():
    object_deps[file_cc] = set([ modify_path (file_cc, tmp=True, strip=cpp_suffix, add=obj_suffix) ])
    for entry in list_headers (file_cc):
      if os.path.abspath(entry).startswith(os.path.abspath(lib_dir)):
        continue
      if 'M' in file_flags[entry]:
        object_deps[file_cc] = object_deps[file_cc].union ([ modify_path (entry, tmp=True, strip=h_suffix, add=moc_obj_suffix) ])
      entry_cc = entry[:-len(h_suffix)] + cpp_suffix
      if os.path.exists (entry_cc):
        object_deps[file_cc] = object_deps[file_cc].union (list_cmd_deps(entry_cc))
      if os.path.basename (entry) == 'icons'+h_suffix:
        object_deps[file_cc].add (modify_path (entry, tmp=True, strip=h_suffix, add=obj_suffix))
    if file_cc not in file_flags:
      file_flags[file_cc] = ''
    for entry in headers[file_cc]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file_cc]:
          file_flags[file_cc] += c

  return object_deps[file_cc]



def list_lib_deps ():
  deps = set()
  for root, dummy_dirs, files in os.walk (lib_dir):
    for current_file in files:
      if current_file[0] == '.':
        continue
      if current_file.endswith (cpp_suffix):
        deps.add (modify_path (os.path.join (root, current_file), tmp=True, strip=cpp_suffix, add=obj_suffix))
  return deps



def build_next ():
  global todo, lock, stop, main_cindex
  total_count = len(todo)
  cindex = 0
  formatstr = '({:>'+str(len(str(total_count)))+'}/'+str(total_count)+') [{}] {}'

  try:
    while not stop:
      current = None
      lock.acquire()
      if todo:
        for item in todo:
          if todo[item].currently_being_processed:
            continue
          unsatisfied_deps = set(todo[item].deps).intersection (todo.keys())
          if not unsatisfied_deps:
            todo[item].currently_being_processed = True
            current = item
            main_cindex+=1
            cindex = main_cindex
            break
      else:
        stop = max (stop, 1)
      lock.release()

      if stop:
        return
      if current is None:
        time.sleep (0.01)
        continue

      target = todo[current]
      if target.execute(cindex, formatstr):
        target.currently_being_processed = False
        stop = 2
        return

      lock.acquire()
      del todo[current]
      lock.release()

  except Exception:
    stop = 2
    return

  stop = max(stop, 1)



#def start_html (fid, title, left, up, home, right):
#  fid.write ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">\n<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">\n')
#  fid.write ('<title>MRtrix documentation</title>\n<link rel="stylesheet" href="../stylesheet.css" type="text/css" media=screen>\n</head>\n<body>\n\n')
#  fid.write ('<table class=nav>\n<tr>\n<td><a href="' + left + '.html"><img src="../left.png"></a></td>\n')
#  fid.write ('<td><a href="' + up + '.html"><img src="../up.png"></a></td>\n')
#  fid.write ('<td><a href="' + home + '.html"><img src="../home.png"></a></td>\n')
#  fid.write ('<th>' + title + '</th>\n')
#  fid.write ('<td><a href="' + right + '.html"><img src="../right.png"></a></td>\n</tr>\n</table>\n')




def version_from_git (folder):
  try:
    log ('fetching version information from git in folder "' + folder + '": ')
    with open (os.devnull) as devnull:
      tag = subprocess.check_output ([ 'git', 'describe', '--abbrev=0' ], cwd=folder, stderr=devnull).decode(errors='ignore').strip() #pylint: disable=unexpected-keyword-arg
      commit = subprocess.check_output ([ 'git', 'describe', '--abbrev=8', '--dirty', '--always' ], cwd=folder, stderr=devnull).decode(errors='ignore').strip() #pylint: disable=unexpected-keyword-arg
    log (tag + ', ' + commit + '\n')
    return [ tag, commit ]
  except Exception:
    log ('not found\n')
    raise LookupError



def version_from_file (version_file, version_macro):
  log ('fetching version information from "' + version_macro + '" in "' + version_file + '": ')
  try:
    with open (version_file) as fd:
      for line in fd:
        line = line.split ("//")[0]
        if version_macro in line:
          tag = line.split(version_macro)[1].strip().strip('"')
          log (tag + '\n')
          return tag
    if not tag:
      raise NameError
  except Exception:
    log ('not found\n')
    raise LookupError
  return None




def mrtrix_version ():
  try:
    return mrtrix_version.version
  except AttributeError:
    pass

  log ('''
getting MRtrix version from folder "''' + mrtrix_dir[-1] + '"...\n  ')
  try:
    tag = version_from_file (os.path.join (lib_dir, 'version.h'), 'MRTRIX_BASE_VERSION')
  except LookupError:
    fail ('ERROR: failed to read version information from file "' + os.path.join (lib_dir, 'version.h') + '"\n')

  try:
    log ('  ')
    version = version_from_git (mrtrix_dir[-1])
    if version[0] != tag:
      fail ('ERROR: version stored in "' + os.path.join (lib_dir, 'version.h') + '" does not match git tag name\n')
    mrtrix_version.version = version[1]
  except LookupError:
    mrtrix_version.version = tag

  log ('Using MRtrix version "' + mrtrix_version.version + '"\n')
  return mrtrix_version.version




def project_version ():
  try:
    return project_version.version
  except AttributeError:
    pass

  project_version.version = None

  log ('''
getting MRtrix project version from current folder...\n  ''')
  try:
    tag = version_from_file (os.path.join (misc_dir, 'project_version.h'), 'MRTRIX_PROJECT_VERSION')
  except LookupError:
    tag = None

  try:
    log ('  ')
    version = version_from_git ('.')
    if tag and version[0] != tag:
      fail ('ERROR: project version stored in "' + os.path.join (misc_dir, 'project_version.h') + '" does not match tag name\n')
    project_version.version = version[1]
  except LookupError:
    project_version.version = tag

  log ('Using MRtrix project version "' + ( project_version.version  or 'unknown' ) + '"\n')
  return project_version.version





def update_version (version, version_file, template):
  version_file_contents = template.replace ('%%%', version or "unknown")

  try:
    with open (version_file) as fd:
      current_version_file_contents = fd.read()
    if ( version is None ) or ( version_file_contents == current_version_file_contents ):
      log ('version file "' + version_file + '" is up to date\n')
      return
  except (IOError, OSError):
    pass

  log ('version file "' + version_file + '" is out of date - updating\n')
  with open (version_file, 'w') as fd:
    fd.write (version_file_contents)





def update_bash_completion ():
  # Only attempt to generate completion file if POSIX compliant
  if os.name != 'posix':
    return

  # Check whether completion generation script exists
  script_path = os.path.join (mrtrix_dir[-1], 'generate_bash_completion.py')
  completion_path = os.path.join (mrtrix_dir[-1], 'mrtrix_bash_completion')
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping bash completion generation. Could not find script at ' + script_path + '\n'))
    return

  # Check whether both command files and completion file exist and completion file is newer than commands
  if not os.path.isdir (target_bin_dir):
    return

  # Only look at relevant executables in bin file
  commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK)]
  if not commands:
    return

  command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
  if os.path.isfile (completion_path) and time.ctime ( os.path.getmtime(completion_path) ) >= command_freshness_time:
    return

  execute ('[SH] ' + completion_path, [ script_path, "-c", completion_path, "-m", target_bin_dir ] )




def update_user_docs ():
  # Only attempt to update docs if POSIX compliant
  if os.name != 'posix':
    return

  scripts_dir = os.path.abspath(os.path.join (mrtrix_dir[-1], 'docs' ))
  script_path = os.path.join (scripts_dir, 'generate_user_docs.sh')

  # Check whether generate docs script exists
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping user documentation generation. Could not find script at ' + script_path + '\n'))
    return

  # Check whether commands dir exists
  if not os.path.isdir (target_bin_dir):
    return

  # Fetch relevant executables in bin file
  commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]

  # Fetch relevant user docs
  rst_files = []

  for root, dummy_subdirs, files in os.walk (scripts_dir):
    for rst_file in files:
      if rst_file.endswith('.rst'):
        rst_files.append(os.path.join(root, rst_file))

  if not commands:
    return

  if rst_files:
    command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
    docs_freshness_time = min (rst_files, key = lambda x: time.ctime ( os.path.getmtime( x ) ) )
    if docs_freshness_time >= command_freshness_time:
      return

  if execute ('[DOC] generating user documentation (./docs)', [ script_path ], scripts_dir):
    sys.exit (1)


###########################################################################
#                              SCRIPT VERSION                             #
###########################################################################

with open (os.path.join(mrtrix_dir[-1], script_dir, '_version.py'),'w') as vfile:
  vfile.write('__version__ = "'+ mrtrix_version() +'" #pylint: disable=unused-variable\n')


###########################################################################
#                              START BUILDING                             #
###########################################################################


if not targets:
  targets = default_targets()



# get git version info:
update_version (mrtrix_version(), os.path.join (lib_dir, 'version.cpp'), '''
namespace MR {
  namespace App {
    const char* mrtrix_version = "%%%";
    const char* build_date = __DATE__;
  }
}
''')

update_version (mrtrix_version(), os.path.join (mrtrix_dir[-1], misc_dir, 'exec_version.cpp'), '''
namespace MR {
  namespace App {
    extern const char* executable_uses_mrtrix_version;
    void set_executable_uses_mrtrix_version () { executable_uses_mrtrix_version = "%%%"; }
  }
}
''')

if separate_project:
  if not os.path.exists (misc_dir):
    os.mkdir (misc_dir)
  update_version (project_version(), os.path.join (misc_dir, 'project_version.cpp'), '''
namespace MR {
  namespace App {
    extern const char* project_version;
    extern const char* project_build_date;
    void set_project_version () {
      project_version = "%%%";
      project_build_date = __DATE__;
    }
  }
}
''')






log ('''
compiling TODO list...
''')
try:
  [ Entry(item) for item in targets ] # pylint: disable=expression-not-assigned
except TargetException as excp:
  fail ('ERROR: ' + str(excp) + '\n')



# for nogui config, remove GUI elements from targets and todo list:
if nogui:
  nogui_targets = []
  for entry in targets:
    if not is_GUI_target (entry):
      nogui_targets.append (entry)
  targets = nogui_targets

  nogui_todo = {}
  for item in todo:
    if not is_GUI_target (todo[item].name):
      nogui_todo[item] = todo[item]
  todo = nogui_todo





log ('building targets: ' + ' '.join (targets) + os.linesep)


if dependencies==1:
  disp ('''
Printing dependencies for targets:

''')
  for entry in targets:
    todo[entry].display()
  sys.exit (0)
elif dependencies==2:
  disp ('''
Printing dependencies for all files:

''')
  for entry in todo:
    todo[entry].display()
  sys.exit (0)

todo_tmp = {}
for item in todo:
  if todo[item].action != '--' and todo[item].need_rebuild():
    todo_tmp[item] = todo[item]
todo = todo_tmp

log ('TODO list contains ' + str(len(todo)) + ''' items

''')


#for entry in todo.values():
#  entry.display()
try:
  num_processors = int(os.environ['NUMBER_OF_PROCESSORS'])
except (KeyError, TypeError):
  try:
    num_processors = os.sysconf('SC_NPROCESSORS_ONLN')
  except ValueError:
    num_processors = 1

while todo:

  stop = False
  main_cindex = 0
  num_todo_previous = len(todo)

  log ('''

  launching ''' + str(num_processors) + ''' threads

  ''')

  threads = []
  for i in range (1, num_processors): # pylint: disable=unused-variable
    t = threading.Thread (target=build_next)
    t.start()
    threads.append (t)

  build_next()

  for t in threads:
    t.join()

  if not persistent:
    break

  if len(todo) == num_todo_previous:
    disp ('''
stopping despite errors as no jobs completed successfully

''')
    break

  if todo:
    disp ('''
retrying as running in persistent mode

''')
    if error_stream is not None:
      error_stream = ''


# generate development-specific files (if needed)
# i.e. bash completion and user documentation
if not separate_project and stop <= 1:
  if bash_completion:
    update_bash_completion()
  if command_doc:
    update_user_docs()

sys.exit (stop > 1)