Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change option name to 'buildtest buildspec find --directory' for searching for root buildspecs when building cache #1807

Merged
merged 7 commits into from
Aug 15, 2024
4 changes: 2 additions & 2 deletions bash_completion.sh
Original file line number Diff line number Diff line change
Expand Up @@ -413,8 +413,8 @@ _buildtest ()
COMPREPLY=( $( compgen -W "${opts}" -- "${cur}" ) );;
# completion for rest of arguments
*)
local longopts="--buildspec --count --executors --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --root --terse"
local shortopts="-b -e -h -n -p -q -r -t"
local longopts="--buildspec --count --directory --executors --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --terse"
local shortopts="-b -d -e -h -n -p -q -r -t"
local cmds="invalid"

COMPREPLY=( $( compgen -W "${cmds} ${longopts} ${shortopts}" -- "${cur}" ) )
Expand Down
2 changes: 1 addition & 1 deletion buildtest/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1287,7 +1287,7 @@ def buildspec_find_menu(self, buildspec_find_parser):
},
),
(
["--root"],
["-d", "--directory"],
{
"type": str,
"action": "append",
Expand Down
30 changes: 16 additions & 14 deletions buildtest/cli/buildspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def __init__(
rebuild=False,
filterfields=None,
formatfields=None,
roots=None,
directory=None,
header=None,
terse=None,
pager=None,
Expand All @@ -58,7 +58,7 @@ def __init__(
row_count=None,
):
"""The initializer method for BuildspecCache class is responsible for loading and finding buildspecs into buildspec cache. First we
resolve paths to directory where buildspecs will be searched. This can be specified via ``--roots`` option on command line or one can
resolve paths to directory where buildspecs will be searched. This can be specified via ``--directory`` option on command line or one can
specify directory paths in the configuration file. Next we build the cache that contains metadata for each buildspec that will be
written to file. If any filter or format options are specified we check if they are valid and finally display a content of the cache
depending on the argument.
Expand All @@ -70,7 +70,7 @@ def __init__(
rebuild (bool, optional): rebuild the buildspec cache by validating all buildspecs when using ``buildtest buildspec find --rebuild``. Defaults to ``False`` if ``--rebuild`` is not specified
filterfields (str, optional): The filter options specified via ``buildtest buildspec find --filter`` that contains list of key value pairs for filtering buildspecs
formatfields (str, optional): The format options used for formating table. The format option is a comma separated list of format fields specified via ``buildtest buildspec find --format``
roots (list, optional): List of directories to search for buildspecs. This argument contains value of ``buildtest buildspec find --roots``
directory (list, optional): List of directories to search for buildspecs. This argument contains value of ``buildtest buildspec find --directory``
headers (bool, optional): Option to control whether header are printed in terse output. This argument contains value of ``buildtest buildspec find --no-header``
terse (bool, optional): Enable terse mode when printing output. In this mode we don't print output in table format instead output is printed in parseable format. This option can be specified via ``buildtest buildspec find --terse``
color (str, optional): An instance of a string class that selects the color to use when printing table output
Expand Down Expand Up @@ -98,8 +98,10 @@ def __init__(
self.row_count = row_count

# if --root is not specified we set to empty list instead of None
self.roots = (
roots or self.configuration.target_config["buildspecs"].get("root") or []
self.directory = (
directory
or self.configuration.target_config["buildspecs"].get("directory")
or []
)

# list of buildspec directories to search for .yml files
Expand All @@ -117,7 +119,7 @@ def __init__(
"rebuild"
)
# if --root is specified we set rebuild to True
if self.roots:
if self.directory:
self.rebuild = True

self.cache = {}
Expand All @@ -138,20 +140,20 @@ def load_paths(self):
"""Add all paths to search for buildspecs. We read configuration file
and check whether we need to load buildspecs from list of directories.
We check if directories exist, if any fail we don't add them to path.
If no root directories are specified we load the default buildspec roots which are
If no root directories are specified we load buildspecs in
`tutorials <https://github.com/buildtesters/buildtest/tree/devel/tutorials>`_
and `general_tests <https://github.com/buildtesters/buildtest/tree/devel/general_tests>`_ directory.
"""

# if no roots specified we load the default buildspec roots.
if not self.roots:
# if no directory is specified we load the default buildspec.
if not self.directory:
self.paths += BUILDSPEC_DEFAULT_PATH

# for every root buildspec defined in configuration or via --root option,
# we resolve path and if path exist add to self.paths. The path must be a
# directory. If its file, we ignore it
if self.roots:
for root in self.roots:
if self.directory:
for root in self.directory:
path = resolve_path(root, exist=False)
if not os.path.exists(path):
console.print(f"[red]Path: {path} does not exist!")
Expand Down Expand Up @@ -297,8 +299,8 @@ def lookup_buildspec_by_name(self, name):

def build_cache(self):
"""This method will rebuild the buildspec cache file by recursively searching
all .yml files specified by input argument ``paths`` which is a list of directory
roots. The buildspecs are validated and cache file is updated
all .yml files specified by input argument ``paths`` which is a list of directory paths.
The buildspecs are validated and cache file is updated
"""

self.update_cache = {}
Expand Down Expand Up @@ -1402,7 +1404,7 @@ def buildspec_find(args, configuration):
rebuild=args.rebuild,
filterfields=args.filter,
formatfields=args.format,
roots=args.root,
directory=args.directory,
configuration=configuration,
header=args.no_header,
terse=args.terse,
Expand Down
2 changes: 1 addition & 1 deletion buildtest/cli/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def print_buildspec_show():
"buildtest buildspec find --pager", "Paginate output of buildspec cache"
)
table.add_row(
"buildtest buildspec find --root /tmp --rebuild",
"buildtest buildspec find --directory /tmp --rebuild",
"Discover buildspecs in /tmp and rebuild buildspec cache",
)
table.add_row(
Expand Down
6 changes: 3 additions & 3 deletions buildtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,10 +252,10 @@ def setup(args):
# build buildspec cache file automatically if it doesn't exist
if not is_file(BUILDSPEC_CACHE_FILE):
root_buildspecs = []
if hasattr(args, "root"):
root_buildspecs = args.root
if hasattr(args, "directory"):
root_buildspecs = args.directory

BuildspecCache(roots=root_buildspecs, configuration=configuration)
BuildspecCache(directory=root_buildspecs, configuration=configuration)

return system, configuration, buildtest_editor, report_file

Expand Down
2 changes: 1 addition & 1 deletion buildtest/schemas/settings.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@
"$ref": "#/definitions/format"
},
"terse": { "$ref": "#/definitions/terse" },
"root": {
"directory": {
"type": "array",
"items": {
"type": "string"
Expand Down
2 changes: 1 addition & 1 deletion buildtest/settings/aws.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ system:
count: 15
format: name,description
terse: false
root:
directory:
- $BUILDTEST_ROOT/aws_tutorial
report:
count: 25
Expand Down
2 changes: 1 addition & 1 deletion buildtest/settings/aws_oddc_pbs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ system:
count: 15
format: name,description
terse: false
root:
directory:
- $BUILDTEST_ROOT/aws_oddc
report:
count: 25
Expand Down
2 changes: 1 addition & 1 deletion buildtest/settings/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ system:
terse: false

# list of paths to search for buildspecs
#root: ["$BUILDTEST_ROOT/tutorials", "$BUILDTEST_ROOT/examples"]
#directory: ["$BUILDTEST_ROOT/tutorials", "$BUILDTEST_ROOT/examples"]

# options for buildtest report command
report:
Expand Down
2 changes: 1 addition & 1 deletion buildtest/settings/spack_container.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ system:
count: 15
format: name,description
terse: false
root:
directory:
- $BUILDTEST_ROOT/examples
report:
count: 25
Expand Down
2 changes: 1 addition & 1 deletion docs/buildtest_perlmutter.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ Exercise 2: Querying Buildspec Cache

In this exercise you will learn how to use the :ref:`buildspec_interface`. Let's build the cache by running the following::

buildtest buildspec find --root $HOME/buildtest-nersc/buildspecs --rebuild -q
buildtest buildspec find --directory $HOME/buildtest-nersc/buildspecs --rebuild -q

.. todo::

Expand Down
16 changes: 8 additions & 8 deletions docs/configuring_buildtest/overview.rst
Original file line number Diff line number Diff line change
Expand Up @@ -225,21 +225,21 @@ can be overridden by command line option.
Specify Root Directories for searching buildspecs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``root`` property in configuration file
is a list of string types which is used to search for buildspecs. The ``root`` property is not required in configuration file, but it can be a good
idea to set this value if you have a predetermined location where buildspecs are stored.
Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``directory`` property in configuration file
is a list of directories to search for buildspecs. The ``directory`` property is not **required** in configuration file, but it can be a good
idea to set this value if you have a pre-determined location where buildspecs are stored.

You can specify the root path via command line ``buildtest buildspec find --root <dir1> --root <dir2>`` which will override the configuration value. In a
You can specify the directory path via command line ``buildtest buildspec find --directory <dir1> --directory <dir2>`` which will override the configuration value. In a
practical situation, you will want to write your buildspecs in a separate repository which you can clone in your filesystem. Let's say they are cloned in
your $HOME directory named **$HOME/buildtest-examples**. You have one of two options, one you can specify the root path in configuration file as shown below:
your **$HOME** directory named **$HOME/buildtest-examples**. You have one of two options, one you can specify the path in configuration file as shown below:

.. code-block:: yaml

buildspecs:
root: [ $HOME/buildtest-examples ]
directory: [ $HOME/buildtest-examples ]

This above configuration will instruct buildtest to search for buildspecs in ``$HOME/buildtest-examples`` directory, and you won't
have to specify the ``--root`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--root`` everytime
This configuration will instruct buildtest to search for buildspecs in ``$HOME/buildtest-examples`` directory, and you won't
have to specify the ``--directory`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--directory`` everytime
you need to build the cache. If neither is specified, buildtest will load the default buildspecs which are **$BUILDTEST_ROOT/tutorials** and
**$BUILDTEST_ROOT/general_tests**.

Expand Down
14 changes: 7 additions & 7 deletions docs/gettingstarted/buildspecs_interface.rst
Original file line number Diff line number Diff line change
Expand Up @@ -92,21 +92,21 @@ or pass them via command line.
buildtest will search buildspecs in :ref:`buildspecs root <buildspec_roots>` defined in your configuration,
which is a list of directory paths to search for buildspecs.
If you want to load buildspecs from a directory path, you can specify a directory
via ``--root`` option in the format: ``buildtest buildspec find --root <path>``.
buildtest will rebuild cache when `--root` option is specified. Note that to rebuild cache you typically
need to pass **--rebuild** option but that is not required when using **--root** option because we want
via ``--directory`` option in the format: ``buildtest buildspec find --directory <path>``.
buildtest will rebuild cache when `--directory` option is specified. Note that to rebuild cache you typically
need to pass **--rebuild** option but that is not required when using **--directory** option because we want
buildtest to load buildspecs into cache.

The **--root** option must be path to a directory, if you specify a file path then buildtest will report an error message similar
The **--directory** option must be path to a directory, if you specify a file path then buildtest will report an error message similar
to one below

.. dropdown:: ``buildtest buildspec find --root $BUILDTEST_ROOT/README.rst``
.. dropdown:: ``buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst``
:color: warning

.. command-output:: buildtest buildspec find --root $BUILDTEST_ROOT/README.rst
.. command-output:: buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst
:returncode: 1

If you want to specify multiple root paths you can specify the **--root** options multiple times.
If you want to specify multiple root paths you can specify the **--directory** options multiple times.

Let's rebuild the cache again by running ``buildtest buildspec find`` which will load the default buildspecs into the cache

Expand Down
41 changes: 0 additions & 41 deletions tests/buildsystem/spack_container.yml

This file was deleted.

7 changes: 4 additions & 3 deletions tests/buildsystem/test_spack.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
from buildtest.cli.build import BuildTest
from buildtest.cli.buildspec import BuildspecCache
from buildtest.config import SiteConfiguration

here = os.path.dirname(os.path.abspath(__file__))
from buildtest.defaults import BUILDTEST_ROOT


@pytest.mark.spack
Expand All @@ -21,7 +20,9 @@ def test_spack_examples():
)

configuration = SiteConfiguration(
settings_file=os.path.join(here, "spack_container.yml")
settings_file=os.path.join(
BUILDTEST_ROOT, "buildtest", "settings", "spack_container.yml"
)
)
configuration.detect_system()
configuration.validate()
Expand Down
6 changes: 4 additions & 2 deletions tests/cli/test_buildspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,12 +354,14 @@ def test_buildspec_find_roots():
os.path.join(BUILDTEST_ROOT, "tutorials"),
]
# buildtest buildspec find --root $BUILDTEST_ROOT/tests/buildsystem --root $BUILDTEST_ROOT/tutorials
BuildspecCache(roots=root_buildspecs, configuration=configuration, rebuild=False)
BuildspecCache(
directory=root_buildspecs, configuration=configuration, rebuild=False
)

with pytest.raises(BuildTestError):
# buildtest buildspec find --root $BUILDTEST_ROOT/README.rst --root $BUILDTEST_ROOT/environment.yml
BuildspecCache(
roots=[
directory=[
os.path.join(BUILDTEST_ROOT, "README.rst"),
os.path.join(BUILDTEST_ROOT, "tutorials", "environment.yml"),
],
Expand Down
2 changes: 1 addition & 1 deletion tests/settings/torque.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ system:
count: 15
format: name,description
terse: false
root:
directory:
- $BUILDTEST_ROOT/aws_oddc
report:
count: 25
Expand Down
Loading