From 4082b143afe222e2944e0b40ec306cbd2d4918f7 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Mon, 31 Aug 2020 17:29:02 +0200 Subject: [PATCH 1/6] Introduce spack-fairsoft extension * Add extension skeleton with * first implementations of * spack fairsoft avail * spack fairsoft setup --- .../spack-fairsoft/fairsoft/__init__.py | 135 ++++++++++++ .../spack-fairsoft/fairsoft/cmd/fairsoft.py | 203 ++++++++++++++++++ thisfairsoft.sh | 105 ++------- 3 files changed, 359 insertions(+), 84 deletions(-) create mode 100644 extensions/spack-fairsoft/fairsoft/__init__.py create mode 100644 extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py diff --git a/extensions/spack-fairsoft/fairsoft/__init__.py b/extensions/spack-fairsoft/fairsoft/__init__.py new file mode 100644 index 000000000..928a3f0b6 --- /dev/null +++ b/extensions/spack-fairsoft/fairsoft/__init__.py @@ -0,0 +1,135 @@ +# Copyright 2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright 2020 GSI Helmholtz Centre for Heavy Ion Research GmbH, Darmstadt +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Module that represents a FairSoft git repo + +Provides useful infos and operations on it: + +root_dir Abs. path to the root of the FairSoft repo +env_dir Abs. path to /env directory +repos_dir Abs. path to /repos directory + +configure_repos() Enforce correct repo config. +get_distros() Return list of '.' distro names. +manage_site_config_dir(config_dir) + Manage symlinks from spack site config dir to given config_dir entries. +""" + +import os + +import llnl.util.tty as tty +import spack.paths +from spack.config import config +from spack.repo import Repo, RepoPath + +root_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), *([os.pardir] * 3))) +env_dir = os.path.join(root_dir, 'env') +repos_dir = os.path.join(root_dir, 'repos') + + +def _repo_eq(self, other): + """TODO: Consider upstreaming as spack.repo.Repo.__eq__()""" + return self.root == other.root + + +def _repo_path_eq(self, other): + """TODO: Consider upstreaming as spack.repo.RepoPath.__eq__()""" + return self.repos == other.repos + + +Repo.__eq__ = _repo_eq # monkeypatch spack.repo.Repo.__eq__() +RepoPath.__eq__ = _repo_path_eq # monkeypatch spack.repo.RepoPath.__eq__() + + +def _update_repos_section(scope, expected): + """Update the repos section in given scope to match expected values if needed + + Return True if an update was performed. + """ + actual = config.get('repos', scope=scope) or [] + if not actual == expected: + tty.debug('{}:{} not equal to {} -> updating ...'.format( + scope, 'repos', expected)) + config.update_config('repos', expected, scope=scope) + return True + return False + + +def configure_repos(): + """Enforce correct repo config. + + 0: fairsoft + 1: fairsoft-backports + 2: builtin + + If expected repo config is not present, set it. + + Return True if a config update was performed. + """ + fairsoft_path = os.path.join(repos_dir, 'fairsoft') + fairsoft_backports_path = os.path.join(repos_dir, 'fairsoft-backports') + builtin_path = '$spack/var/spack/repos/builtin' + + expected = RepoPath(fairsoft_path, fairsoft_backports_path, builtin_path) + # We accept if the repo config correct, no matter from which sections it comes + actual = RepoPath(*(config.get('repos') or [])) + + res = False + if not actual == expected: + # Now we have to write it somewhere, so we hardcode each section + res = _update_repos_section('defaults', [builtin_path]) or res + res = _update_repos_section('system', []) or res + res = _update_repos_section( + 'site', [fairsoft_path, fairsoft_backports_path]) or res + res = _update_repos_section('user', []) or res + return res + + +def get_distros(): + """Return list of '.' distro names. + + List is generated from the directory names in `/env//`. + """ + envs = [] + for release in os.listdir(env_dir): + for variant in os.listdir(os.path.join(env_dir, release)): + envs.append('{}.{}'.format(release.lower(), variant.lower())) + return envs + + +def manage_site_config_dir(config_dir): + """Manage symlinks from spack site config dir to given config_dir entries.""" + fairsoft = os.path.abspath(config_dir) + site = os.path.join(spack.paths.etc_path, 'spack') + tty.debug('Managing symlinks from {} to {}'.format(site, fairsoft)) + + def compute_symlink_and_target(site, fairsoft, entry): + return [ + os.path.join(site, entry), + os.path.relpath(os.path.join(fairsoft, entry), start=site) + ] + + # Remove dangling and unexpected symlinks + for entry in os.listdir(site): + symlink, target = compute_symlink_and_target(site, fairsoft, entry) + if os.path.islink(symlink): + readlink = os.readlink(symlink) + if readlink != target: + tty.debug('Removing unexpected symlink {} to {} (expected {})'. + format(symlink, readlink, target)) + os.remove(symlink) + if not os.path.exists(os.path.abspath(os.path.join(site, + readlink))): + tty.debug('Removing dangling symlink {} to {}'.format( + symlink, readlink)) + os.remove(symlink) + + # Add missing symlinks + for entry in os.listdir(fairsoft): + symlink, target = compute_symlink_and_target(site, fairsoft, entry) + if not (os.path.islink(symlink) and os.readlink(symlink) == target): + tty.debug('Add missing symlink {} to {}'.format(symlink, target)) + os.symlink(target, symlink) diff --git a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py new file mode 100644 index 000000000..8fe7ae2a7 --- /dev/null +++ b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py @@ -0,0 +1,203 @@ +# Copyright 2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright 2020 GSI Helmholtz Centre for Heavy Ion Research GmbH, Darmstadt +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import sys + +import llnl.util.tty as tty +from llnl.util.tty.colify import colify +from spack.cmd.clean import clean +from spack.cmd.compiler import compiler_find +from spack.cmd.repo import repo_list +import spack.extensions.fairsoft as ext +from spack.util.executable import which + +description = 'manage FairSoft distros' +section = 'FairSoft distro' +level = 'short' + + +def _spack_clean_ms(): + """Call `spack clean -ms`""" + args = argparse.Namespace() + args.specs = False + args.stage = True + args.downloads = False + args.failures = False + args.misc_cache = True + args.python_cache = False + args.all = False + clean(None, args) + + +def _spack_compiler_find_scope_site(): + """Call `spack compiler find --scope site`""" + args = argparse.Namespace() + args.add_paths = None + args.scope = 'site' + compiler_find(args) + + +def _spack_repo_list(): + """Call `spack repo list`""" + args = argparse.Namespace() + args.scope = None + repo_list(args) + + +def avail(_args): + """show available FairSoft distros""" + distros = sorted(ext.get_distros()) + + if sys.stdout.isatty(): + if not distros: + tty.msg('No distros') + else: + tty.msg('{} distros'.format(len(distros))) + + colify(distros, indent=4, cols=1) + + +def info(_args): + """show info on a FairSoft distro""" + raise NotImplementedError('NOT YET IMPLEMENTED') + + +def install(_args): + """install a FairSoft distro""" + raise NotImplementedError('NOT YET IMPLEMENTED') + + +def list(_args): + """list installed FairSoft distros""" + raise NotImplementedError('NOT YET IMPLEMENTED') + + +def setup(args): + """setup spack for FairSoft + + Tasks performed: + 1. `git submodule update --init` + 2. `spack compiler find --scope site` + 3. Manage fairsoft site config dir + 4. Check/Update repo config + 5. `spack clean -ms` + + Do not run in parallel to another running spack command! + """ + if not args.skip_git: + git = which('git', required=True) + git('-C', ext.root_dir, 'submodule', 'update', '--init') + if not args.skip_compiler: + _spack_compiler_find_scope_site() + if args.config_dir: + ext.manage_site_config_dir(args.config_dir[0]) + if not args.skip_repos and ext.configure_repos(): + tty.info('Updated repo config') + _spack_repo_list() + if not args.skip_clean: + _spack_clean_ms() + tty.info('Report problems at https://github.com/FairRootGroup/FairSoft/issues/new') + + +def view(_args): + """manage symlink spack views on a FairSoft distro""" + raise NotImplementedError('NOT YET IMPLEMENTED') + + +# Map subcommand to action function +_action = { + 'avail': avail, + 'info': info, + 'install': install, + 'list': list, + 'setup': setup, + 'view': view +} + + +def _trim_docstring(docstring): + """Dedent docstrings as specified by PEP-0257. + + Reference implementation from https://www.python.org/dev/peps/pep-0257/ + + `sys.maxint` replaced with `sys.maxsize`. + """ + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +def _add_cmd(parsers, cmd): + """Helper to add subcmd parser + + Uses the summary line of the action function's docstring + as short help text and the body docstring as long + description. + """ + docstring = _action[cmd].__doc__ + summary = _trim_docstring(docstring.splitlines()[0]) + body = '\n'.join(_trim_docstring(docstring).splitlines()[1:]) + return parsers.add_parser(cmd, + help=summary, + description=summary, + epilog=body) + + +def setup_parser(parser): + """Define CLI subcommands and options""" + subcmds = parser.add_subparsers(metavar='SUBCOMMAND', + dest='fairsoft_subcommand') + _add_cmd(subcmds, 'avail') + + _add_cmd(subcmds, 'info') + + _add_cmd(subcmds, 'install') + + _add_cmd(subcmds, 'list') + + setup_cmd = _add_cmd(subcmds, 'setup') + setup_cmd.add_argument('--skip-clean', + action='store_true', + help='do not run `spack clean -ms`') + setup_cmd.add_argument('--skip-compiler', + action='store_true', + help='do not run `spack compiler find --scope site`') + setup_cmd.add_argument('--skip-git', + action='store_true', + help='do not run `git submodule update --init`') + setup_cmd.add_argument('--skip-repos', + action='store_true', + help='do not configure repos') + setup_cmd.add_argument('--config-dir', nargs=1, default=None, + help='symlink provided config dir') + + _add_cmd(subcmds, 'view') + + +def fairsoft(_parser, args): + """Dispatch to SUBCOMMAND action function""" + _action[args.fairsoft_subcommand](args) diff --git a/thisfairsoft.sh b/thisfairsoft.sh index a7127245d..66a802a2f 100755 --- a/thisfairsoft.sh +++ b/thisfairsoft.sh @@ -1,98 +1,35 @@ #!/bin/bash fairsoft_basedir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -fairsoft_spackdir=spack -fairsoft_configdir=../../../config +fairsoft_spackdir=${fairsoft_basedir}/spack +fairsoft_configdir=${fairsoft_basedir}/config -if [ "$1" = "--setup" ] -then - fairsoft_do_setup=true -else - fairsoft_do_setup=false -fi - -if $fairsoft_do_setup && command -v git >/dev/null 2>&1 -then - (cd "$fairsoft_basedir" && git submodule update --init) -fi - -. "${fairsoft_basedir}/${fairsoft_spackdir}/share/spack/setup-env.sh" - -if $fairsoft_do_setup -then - spack compiler find -fi - -( -cd "${fairsoft_basedir}/${fairsoft_spackdir}/etc/spack/${fairsoft_configdir}" -for config_entry in * -do - case "$config_entry" in - *~) - continue - ;; - .*) - continue - ;; - esac - fairsoft_symlink_from="${fairsoft_basedir}/${fairsoft_spackdir}/etc/spack/${config_entry}" - if [ "!" -e "$fairsoft_symlink_from" ] && [ "!" -L "$fairsoft_symlink_from" ] - then - if $fairsoft_do_setup - then - ln -s "${fairsoft_configdir}/${config_entry}" \ - "$fairsoft_symlink_from" - else - echo '!!!' >&2 - echo '!!! ========================== /!\ WARNING ==========================' >&2 - echo "!!! $fairsoft_symlink_from" >&2 - echo "!!! should be a symlink into the FairSoft config directory" >&2 - echo "!!! Consider running the setup once:" >&2 - echo "!!! --> source ${BASH_SOURCE[0]} --setup" >&2 - fi - else - fairsoft_readlink="$(readlink "$fairsoft_symlink_from")" - if [ "$fairsoft_readlink" != "${fairsoft_configdir}/${config_entry}" ] - then - echo '!!!' >&2 - echo '!!! ========================== /!\ WARNING ==========================' >&2 - echo "!!! $fairsoft_symlink_from" >&2 - echo "!!! does not point to the right target" >&2 - echo "!!! It should be a symlink to:" >&2 - echo "!!! ${fairsoft_configdir}/${config_entry}" >&2 - echo "!!! Currently either not a symlink or points to:" >&2 - echo "!!! $fairsoft_readlink" >&2 - fi - fi -done -) +spack_log() { + spack python -c "import llnl.util.tty as tty; tty.$1('$2')" +} -fairsoft_repo() { - if [ "$(spack repo list | sed -n -e "/^$1 / { s/^[^ ]* *//; p; }")" != "${fairsoft_basedir}/repos/$2" ] - then - if $fairsoft_do_setup - then - spack repo add --scope site "${fairsoft_basedir}/repos/$2" - else - echo '!!!' >&2 - echo '!!! ========================== /!\ WARNING ==========================' >&2 - echo "!!! spack repo $1 ($2) missing" >&2 - echo "!!! Consider running the setup once:" >&2 - echo "!!! --> source ${BASH_SOURCE[0]} --setup" >&2 - fi - fi +register_spack_extension() { + ext="$(realpath -e ${fairsoft_basedir}/extensions/spack-$1)" + if [ -d "$ext" ] + then + if spack python -c "from spack.config import get; exts = get('config:extensions') or []; exit(0) if '$ext' not in exts else exit(1)" + then + spack_log info "Registering extension ${ext}" + spack config --scope site add "config:extensions:${ext}" + fi + fi } -fairsoft_repo "fairsoft_backports" "fairsoft-backports" -fairsoft_repo "fairsoft" "fairsoft" +. "${fairsoft_spackdir}/share/spack/setup-env.sh" -if $fairsoft_do_setup +if [ "$1" = "--setup" ] then - spack clean -ms + register_spack_extension "fairsoft" + spack fairsoft setup --config-dir ${fairsoft_configdir} fi -unset -f fairsoft_repo +unset -f spack_log +unset -f register_spack_extension unset fairsoft_basedir unset fairsoft_spackdir unset fairsoft_configdir -unset fairsoft_do_setup From 7c214bc9af37bf62e79a165e8348de0de5058228 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Sun, 20 Sep 2020 17:01:55 +0200 Subject: [PATCH 2/6] fairsoft-ext: Implement info subcommand --- env/dev/sim/info.yaml | 2 ++ env/dev/sim_mt/info.yaml | 2 ++ env/dev/sim_mt_headless/info.yaml | 2 ++ env/jun19/sim/info.yaml | 2 ++ env/jun19/sim_mt/info.yaml | 2 ++ .../spack-fairsoft/fairsoft/__init__.py | 25 ++++++++++++-- .../spack-fairsoft/fairsoft/cmd/fairsoft.py | 34 +++++++++++++++++-- 7 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 env/dev/sim/info.yaml create mode 100644 env/dev/sim_mt/info.yaml create mode 100644 env/dev/sim_mt_headless/info.yaml create mode 100644 env/jun19/sim/info.yaml create mode 100644 env/jun19/sim_mt/info.yaml diff --git a/env/dev/sim/info.yaml b/env/dev/sim/info.yaml new file mode 100644 index 000000000..e8e731e11 --- /dev/null +++ b/env/dev/sim/info.yaml @@ -0,0 +1,2 @@ +info: + description: FairRoot v18.4.0 + full simulation dependencies (single-threaded Geant4) diff --git a/env/dev/sim_mt/info.yaml b/env/dev/sim_mt/info.yaml new file mode 100644 index 000000000..6ec88c4ad --- /dev/null +++ b/env/dev/sim_mt/info.yaml @@ -0,0 +1,2 @@ +info: + description: FairRoot v18.4.0 + full simulation dependencies (multi-threaded Geant4) diff --git a/env/dev/sim_mt_headless/info.yaml b/env/dev/sim_mt_headless/info.yaml new file mode 100644 index 000000000..8287b4a17 --- /dev/null +++ b/env/dev/sim_mt_headless/info.yaml @@ -0,0 +1,2 @@ +info: + description: FairRoot v18.4.0 + simulation dependencies (multi-threaded Geant4) without Graphics packages diff --git a/env/jun19/sim/info.yaml b/env/jun19/sim/info.yaml new file mode 100644 index 000000000..929182067 --- /dev/null +++ b/env/jun19/sim/info.yaml @@ -0,0 +1,2 @@ +info: + description: FairRoot v18.2.1 + full simulation dependencies (single-threaded Geant4) diff --git a/env/jun19/sim_mt/info.yaml b/env/jun19/sim_mt/info.yaml new file mode 100644 index 000000000..1d4603b9c --- /dev/null +++ b/env/jun19/sim_mt/info.yaml @@ -0,0 +1,2 @@ +info: + description: FairRoot v18.2.1 + full simulation dependencies (multi-threaded Geant4) diff --git a/extensions/spack-fairsoft/fairsoft/__init__.py b/extensions/spack-fairsoft/fairsoft/__init__.py index 928a3f0b6..121dfec64 100644 --- a/extensions/spack-fairsoft/fairsoft/__init__.py +++ b/extensions/spack-fairsoft/fairsoft/__init__.py @@ -11,8 +11,9 @@ env_dir Abs. path to /env directory repos_dir Abs. path to /repos directory -configure_repos() Enforce correct repo config. -get_distros() Return list of '.' distro names. +configure_repos() Enforce correct repo config. +get_distros() Return list of '.' distro names. +get_distro_info(distro) Return an info dictionary about given distro. manage_site_config_dir(config_dir) Manage symlinks from spack site config dir to given config_dir entries. """ @@ -20,6 +21,7 @@ import os import llnl.util.tty as tty +from ruamel.yaml.main import safe_load import spack.paths from spack.config import config from spack.repo import Repo, RepoPath @@ -100,6 +102,25 @@ def get_distros(): return envs +def get_distro_info(distro): + """Return an info dictionary about given distro.""" + release, variant = distro.split('.') + info_file = os.path.join(env_dir, release, variant, 'info.yaml') + data = {'info': {}} + if os.path.exists(info_file): + tty.debug('Reading distro info file {}'.format(info_file)) + with open(info_file) as f: + data = safe_load(f) + + info = data['info'] + return { + 'name': distro, + 'release': release, + 'variant': variant, + 'description': info['description'] if 'description' in info else 'None' + } + + def manage_site_config_dir(config_dir): """Manage symlinks from spack site config dir to given config_dir entries.""" fairsoft = os.path.abspath(config_dir) diff --git a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py index 8fe7ae2a7..d54a61989 100644 --- a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py +++ b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py @@ -8,12 +8,15 @@ import sys import llnl.util.tty as tty +import llnl.util.tty.color as color from llnl.util.tty.colify import colify from spack.cmd.clean import clean from spack.cmd.compiler import compiler_find from spack.cmd.repo import repo_list import spack.extensions.fairsoft as ext +from spack.spec import version_color from spack.util.executable import which +from textwrap import wrap description = 'manage FairSoft distros' section = 'FairSoft distro' @@ -61,9 +64,31 @@ def avail(_args): colify(distros, indent=4, cols=1) -def info(_args): +def info(args): """show info on a FairSoft distro""" - raise NotImplementedError('NOT YET IMPLEMENTED') + cl_header = '@*b' + cl_plain = '@.' + cl_version = version_color + info = ext.get_distro_info(args.distro) + + color.cprint('{}Distro:{} {}'.format(cl_header, cl_plain, info['name'])) + color.cprint('') + color.cprint('{}Description:{}'.format(cl_header, cl_plain)) + colify(wrap(info['description'], width=70), indent=4, cols=1) + color.cprint('') + color.cprint('{}Release:{} {}{}{}'.format(cl_header, cl_plain, cl_version, + info['release'], cl_plain)) + color.cprint('') + color.cprint('{}Variant:{} {}'.format(cl_header, cl_plain, + info['variant'])) + color.cprint('') + color.cprint('{}Packages:{} {}'.format(cl_header, cl_plain, 'TODO')) + color.cprint('') + color.cprint('{}Prerequisites installed:{} {}'.format( + cl_header, cl_plain, 'TODO')) + color.cprint('') + color.cprint('{}Installed:{} {}'.format(cl_header, cl_plain, 'TODO')) + color.cprint('') def install(_args): @@ -173,7 +198,10 @@ def setup_parser(parser): dest='fairsoft_subcommand') _add_cmd(subcmds, 'avail') - _add_cmd(subcmds, 'info') + info_cmd = _add_cmd(subcmds, 'info') + info_cmd.add_argument('distro', + choices=ext.get_distros(), + help='name of distro to print details about') _add_cmd(subcmds, 'install') From 889554857a849b50b736bdcb920f1f7ae71f8be2 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Sun, 20 Sep 2020 19:44:48 +0200 Subject: [PATCH 3/6] fairsoft-ext: Implement install subcommand --- .../spack-fairsoft/fairsoft/__init__.py | 53 ++++++++++++++++++- .../spack-fairsoft/fairsoft/cmd/fairsoft.py | 23 ++++++-- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/extensions/spack-fairsoft/fairsoft/__init__.py b/extensions/spack-fairsoft/fairsoft/__init__.py index 121dfec64..97ef19ff3 100644 --- a/extensions/spack-fairsoft/fairsoft/__init__.py +++ b/extensions/spack-fairsoft/fairsoft/__init__.py @@ -12,19 +12,22 @@ repos_dir Abs. path to /repos directory configure_repos() Enforce correct repo config. -get_distros() Return list of '.' distro names. +get_available_distros() Return list of '.' distro names. get_distro_info(distro) Return an info dictionary about given distro. manage_site_config_dir(config_dir) Manage symlinks from spack site config dir to given config_dir entries. """ +import functools import os import llnl.util.tty as tty from ruamel.yaml.main import safe_load import spack.paths from spack.config import config +import spack.environment as ev from spack.repo import Repo, RepoPath +from spack.util.executable import which root_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), *([os.pardir] * 3))) @@ -90,7 +93,53 @@ def configure_repos(): return res -def get_distros(): +def create_distro(distro): + """Create given distro by (re-)creating a named environment""" + active_env = ev.get_env({'env': distro}, '') + release, variant = distro.split('.') + env_name = 'fairsoft_{}_{}'.format(release, variant) + env_path = os.path.join(env_dir, release, variant, 'spack.yaml') + commit_file = os.path.join(ev.root(env_name), 'commit_hash') + + if active_env: + tty.die('Found active environment `{}`.'.format(active_env.name), + 'This is not supported by this command.', + 'Please deactivate to continue:', + ' spack env deactivate') + + git = which('git', required=True) + commit_hash = git('-C', root_dir, 'rev-parse', 'HEAD', output=str) + last_commit_hash = None + + if ev.exists(env_name): + if os.path.exists(commit_file): + with open(commit_file, 'r') as f: + last_commit_hash = f.read() + if last_commit_hash != commit_hash: + tty.debug('Removing environment `{}` because \ + {} ({}) != {} (current `git rev-parse HEAD`)'.format( + env_name, last_commit_hash, commit_file, commit_hash)) + env = ev.read(env_name) + env.destroy() + + env = None + if ev.exists(env_name): + env = ev.read(env_name) + else: + tty.debug('Creating environment `{}` from {}'.format(env_name, + env_path)) + env = ev.create(env_name, env_path) + with env.write_transaction(): + env.write() + tty.debug('Writing commit_hash {} to {}'.format(commit_hash, commit_file)) + with open(commit_file, 'w') as f: + f.write(commit_hash) + + return env + + +@functools.lru_cache(maxsize=100, typed=False) +def get_available_distros(): """Return list of '.' distro names. List is generated from the directory names in `/env//`. diff --git a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py index d54a61989..2125fa10c 100644 --- a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py +++ b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py @@ -13,6 +13,7 @@ from spack.cmd.clean import clean from spack.cmd.compiler import compiler_find from spack.cmd.repo import repo_list +import spack.environment as ev import spack.extensions.fairsoft as ext from spack.spec import version_color from spack.util.executable import which @@ -53,7 +54,7 @@ def _spack_repo_list(): def avail(_args): """show available FairSoft distros""" - distros = sorted(ext.get_distros()) + distros = sorted(ext.get_available_distros()) if sys.stdout.isatty(): if not distros: @@ -91,9 +92,18 @@ def info(args): color.cprint('') -def install(_args): +def install(args): """install a FairSoft distro""" - raise NotImplementedError('NOT YET IMPLEMENTED') + env = ext.create_distro(args.distro) + with env.write_transaction(): + concretized_specs = env.concretize() + ev.display_specs(concretized_specs) + env.write(regenerate_views=False) + + tty.msg("Installing environment %s" % env.name) + env.install_all() + with env.write_transaction(): + env.regenerate_views() def list(_args): @@ -200,10 +210,13 @@ def setup_parser(parser): info_cmd = _add_cmd(subcmds, 'info') info_cmd.add_argument('distro', - choices=ext.get_distros(), + choices=ext.get_available_distros(), help='name of distro to print details about') - _add_cmd(subcmds, 'install') + install_cmd = _add_cmd(subcmds, 'install') + install_cmd.add_argument('distro', + choices=ext.get_available_distros(), + help='name of distro to install') _add_cmd(subcmds, 'list') From 2f2631ae42aafd30548ab86d3a4a648013c195e9 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Mon, 21 Sep 2020 07:05:55 +0200 Subject: [PATCH 4/6] fairsoft-ext: Factor out distro env name rule --- extensions/spack-fairsoft/fairsoft/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extensions/spack-fairsoft/fairsoft/__init__.py b/extensions/spack-fairsoft/fairsoft/__init__.py index 97ef19ff3..9cffb8e79 100644 --- a/extensions/spack-fairsoft/fairsoft/__init__.py +++ b/extensions/spack-fairsoft/fairsoft/__init__.py @@ -13,6 +13,7 @@ configure_repos() Enforce correct repo config. get_available_distros() Return list of '.' distro names. +get_distro_env_name(distro) Compute the environment name for the given distro name. get_distro_info(distro) Return an info dictionary about given distro. manage_site_config_dir(config_dir) Manage symlinks from spack site config dir to given config_dir entries. @@ -97,7 +98,7 @@ def create_distro(distro): """Create given distro by (re-)creating a named environment""" active_env = ev.get_env({'env': distro}, '') release, variant = distro.split('.') - env_name = 'fairsoft_{}_{}'.format(release, variant) + env_name = get_distro_env_name(distro) env_path = os.path.join(env_dir, release, variant, 'spack.yaml') commit_file = os.path.join(ev.root(env_name), 'commit_hash') @@ -170,6 +171,12 @@ def get_distro_info(distro): } +@functools.lru_cache(maxsize=100, typed=False) +def get_distro_env_name(distro): + """Compute the environment name for the given distro name.""" + return 'fairsoft_{}_{}'.format(*distro.split('.')) + + def manage_site_config_dir(config_dir): """Manage symlinks from spack site config dir to given config_dir entries.""" fairsoft = os.path.abspath(config_dir) From a1bcc8881a84745b057182d7230786f79c87da64 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Mon, 21 Sep 2020 09:00:31 +0200 Subject: [PATCH 5/6] fairsoft-ext: Implement list subcommand --- .../spack-fairsoft/fairsoft/cmd/fairsoft.py | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py index 2125fa10c..dee1d7cb0 100644 --- a/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py +++ b/extensions/spack-fairsoft/fairsoft/cmd/fairsoft.py @@ -9,7 +9,8 @@ import llnl.util.tty as tty import llnl.util.tty.color as color -from llnl.util.tty.colify import colify +from llnl.util.tty.colify import colify, colify_table +from llnl.util.tty.color import colorize, cescape from spack.cmd.clean import clean from spack.cmd.compiler import compiler_find from spack.cmd.repo import repo_list @@ -108,7 +109,53 @@ def install(args): def list(_args): """list installed FairSoft distros""" - raise NotImplementedError('NOT YET IMPLEMENTED') + available = ext.get_available_distros() + created = sorted([d for d in available if ev.exists(ext.get_distro_env_name(d))]) + + table = [] + for distro in created: + row = [] + row.append(distro) + + env = ev.read(ext.get_distro_env_name(distro)) + # Make sure it is concretized + with env.write_transaction(): + env.concretize() + env.write(regenerate_views=False) + + # Count all and installed packages + all = set() + installed = set() + for spec in env.all_specs(): + nodes = spec.traverse(root=False, cover='nodes') + for node in nodes: + all.add(node) + if node.package.installed: + installed.add(node) + + num_all = len(all) + num_installed = len(installed) + if (num_installed < num_all): + row.append(colorize('@b[{}/{} installed packages]@.'.format( + num_installed, num_all), color=sys.stdout.isatty())) + else: + row.append(colorize('@b[{} installed packages]@.'.format( + num_installed), color=sys.stdout.isatty())) + + row.append(colorize('@m{}@.'.format(env.all_specs()[0].architecture), + color=sys.stdout.isatty())) + row.append(colorize('@g{}@.'.format(cescape(env.all_specs()[0].compiler)), + color=sys.stdout.isatty())) + table.append(row) + + if sys.stdout.isatty(): + if not table: + tty.msg('No distros installed') + else: + tty.msg('{} distros installed'.format(len(table))) + + if table: + colify_table(table, indent=4) def setup(args): From 8683a2412ccb89f581d79494fdde37259bc1a12a Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Fri, 25 Sep 2020 13:06:37 +0200 Subject: [PATCH 6/6] fairsoft-ext: Increase log verbosity --- extensions/spack-fairsoft/fairsoft/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/spack-fairsoft/fairsoft/__init__.py b/extensions/spack-fairsoft/fairsoft/__init__.py index 9cffb8e79..83def108e 100644 --- a/extensions/spack-fairsoft/fairsoft/__init__.py +++ b/extensions/spack-fairsoft/fairsoft/__init__.py @@ -195,12 +195,12 @@ def compute_symlink_and_target(site, fairsoft, entry): if os.path.islink(symlink): readlink = os.readlink(symlink) if readlink != target: - tty.debug('Removing unexpected symlink {} to {} (expected {})'. - format(symlink, readlink, target)) + tty.msg('Removing unexpected symlink {} to {} (expected {})'.format( + symlink, readlink, target)) os.remove(symlink) if not os.path.exists(os.path.abspath(os.path.join(site, readlink))): - tty.debug('Removing dangling symlink {} to {}'.format( + tty.msg('Removing dangling symlink {} to {}'.format( symlink, readlink)) os.remove(symlink) @@ -208,5 +208,5 @@ def compute_symlink_and_target(site, fairsoft, entry): for entry in os.listdir(fairsoft): symlink, target = compute_symlink_and_target(site, fairsoft, entry) if not (os.path.islink(symlink) and os.readlink(symlink) == target): - tty.debug('Add missing symlink {} to {}'.format(symlink, target)) + tty.msg('Add missing symlink {} to {}'.format(symlink, target)) os.symlink(target, symlink)