Skip to content

Commit

Permalink
Introduce new helpers to support BITS_PATH (#897)
Browse files Browse the repository at this point in the history 
Add support for BITS_PATH to have multiple overlayed repositories
  • Loading branch information
@ktf
ktf authored Feb 2, 2025
1 parent 8c8460f commit f150599
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 99 deletions.
35 changes: 30 additions & 5 deletions alibuild_helpers/utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from datetime import datetime
from collections import OrderedDict
from shlex import quote
from typing import Optional

from alibuild_helpers.cmd import getoutput
from alibuild_helpers.git import git
Expand Down Expand Up @@ -282,8 +283,8 @@ def disabledByArchitectureDefaults(arch, defaults, requires):
yield require

def readDefaults(configDir, defaults, error, architecture):
defaultsFilename = "%s/defaults-%s.sh" % (configDir, defaults)
if not exists(defaultsFilename):
defaultsFilename = resolveDefaultsFilename(defaults, configDir)
if not defaultsFilename:
error("Default `%s' does not exists. Viable options:\n%s" %
(defaults or "<no defaults specified>",
"\n".join("- " + basename(x).replace("defaults-", "").replace(".sh", "")
Expand All @@ -305,7 +306,7 @@ def readDefaults(configDir, defaults, error, architecture):
defaultsBody += "\n# Architecture defaults\n" + archBody
return (defaultsMeta, defaultsBody)

# Get the appropriate recipe reader depending on th filename

def getRecipeReader(url, dist=None):
m = re.search(r'^dist:(.*)@([^@]+)$', url)
if m and dist:
Expand Down Expand Up @@ -406,9 +407,30 @@ def parseDefaults(disable, defaultsGetter, log):
overrides[f] = dict(**(v or {}))
return (None, overrides, taps)

def checkForFilename(taps: dict, pkg: str, d: str):
return taps.get(pkg, join(d, f"{pkg}.sh"))

def getPkgDirs(configDir):
configPath = os.environ.get("BITS_PATH", "").rstrip(":") + ":"
pkgDirs = [join(configDir, d) for d in configPath.lstrip(":").split(":")]
return pkgDirs

def resolveFilename(taps: dict, pkg: str, configDir: str):
for d in getPkgDirs(configDir):
filename = checkForFilename(taps, pkg, d)
if os.path.exists(filename):
return (filename, os.path.abspath(d))
return (None, None)

def resolveDefaultsFilename(defaults, configDir) -> Optional[str]:
for d in getPkgDirs(configDir):
filename = join(d, f"defaults-{defaults}.sh")
if os.path.exists(filename):
return filename

def getPackageList(packages, specs, configDir, preferSystem, noSystem,
architecture, disable, defaults, performPreferCheck, performRequirementCheck,
performValidateDefaults, overrides, taps, log, force_rebuild=()):
performValidateDefaults, overrides, taps: dict, log, force_rebuild=()):
systemPackages = set()
ownPackages = set()
failedRequirements = set()
Expand All @@ -429,7 +451,9 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
# "defaults-release" for this to work, since the defaults are a dependency
# and all dependencies' names go into a package's hash.
pkg_filename = ("defaults-" + defaults) if p == "defaults-release" else p.lower()
filename = taps.get(pkg_filename, "%s/%s.sh" % (configDir, pkg_filename))

filename, pkgdir = resolveFilename(taps, pkg_filename, configDir)

err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir))
dieOnError(err, err)
# Unless there was an error, both spec and recipe should be valid.
Expand All @@ -438,6 +462,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
assert(recipe is not None)
dieOnError(spec["package"].lower() != pkg_filename,
"%s.sh has different package field: %s" % (p, spec["package"]))
spec["pkgdir"] = pkgdir

if p == "defaults-release":
# Re-rewrite the defaults' name to "defaults-release". Everything auto-
Expand Down
13 changes: 8 additions & 5 deletions tests/test_deps.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from unittest.mock import patch, MagicMock
from io import StringIO
import os.path

from alibuild_helpers.deps import doDeps
from argparse import Namespace
Expand Down Expand Up @@ -64,12 +65,14 @@ def depsOpen(fn, mode):
outgraph="/tmp/outgraph.pdf",
package="AliRoot",
defaults="release")
def fake_exists(n):
return {"/alidist/aliroot.sh": True}
with patch.object(os.path, "exists", fake_exists):
doDeps(args, MagicMock())

doDeps(args, MagicMock())

# Same check without explicit intermediate dotfile
args.outdot = None
doDeps(args, MagicMock())
# Same check without explicit intermediate dotfile
args.outdot = None
doDeps(args, MagicMock())


if __name__ == '__main__':
Expand Down
76 changes: 40 additions & 36 deletions tests/test_doctor.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import print_function
from unittest.mock import patch, MagicMock
from io import StringIO
import os.path

from alibuild_helpers.doctor import doDoctor
from argparse import Namespace
Expand Down Expand Up @@ -84,42 +85,45 @@ def resetOut():
disable=[],
defaults="release")

# What to call (longer names deprecated in Python 3.5+)
if not hasattr(self, "assertRegex"):
self.assertRegex = self.assertRegexpMatches
self.assertNotRegex = self.assertNotRegexpMatches

# Test: all should go OK (exit with 0)
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["Package1"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 0)

# Test: system dependency not found
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["SysDep"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 1)

# Test: invalid default
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["BreakDefaults"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 2)
self.assertRegex(out["error"].getvalue(), "- its_not_there")

# Test: common defaults
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["TestDef1"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 2)
self.assertRegex(out["banner"].getvalue(), "- common_default")
self.assertNotRegex(out["banner"].getvalue(), "- default1")
self.assertNotRegex(out["banner"].getvalue(), "- default2")
def fake_exists(n):
return {"/alidist/aliroot.sh": True}
with patch.object(os.path, "exists", fake_exists):
# What to call (longer names deprecated in Python 3.5+)
if not hasattr(self, "assertRegex"):
self.assertRegex = self.assertRegexpMatches
self.assertNotRegex = self.assertNotRegexpMatches

# Test: all should go OK (exit with 0)
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["Package1"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 0)

# Test: system dependency not found
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["SysDep"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 1)

# Test: invalid default
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["BreakDefaults"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 2)
self.assertRegex(out["error"].getvalue(), "- its_not_there")

# Test: common defaults
out = resetOut()
with self.assertRaises(SystemExit) as cm:
args.packages=["TestDef1"]
doDoctor(args, MagicMock())
self.assertEqual(cm.exception.code, 2)
self.assertRegex(out["banner"].getvalue(), "- common_default")
self.assertNotRegex(out["banner"].getvalue(), "- default1")
self.assertNotRegex(out["banner"].getvalue(), "- default2")

if __name__ == '__main__':
unittest.main()
24 changes: 14 additions & 10 deletions tests/test_init.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from argparse import Namespace
import os.path as path
import os.path
import unittest
from unittest.mock import call, patch # In Python 3, mock is built-in
from io import StringIO
Expand Down Expand Up @@ -82,17 +83,20 @@ def test_doRealInit(self, mock_read_defaults, mock_open, mock_update_reference,
fetchRepos = False,
architecture = "slc7_x86-64"
)
doInit(args)
self.assertEqual(mock_git.mock_calls, CLONE_EVERYTHING)
mock_path.exists.assert_has_calls([call('.'), call('/sw/MIRROR'), call('/alidist'), call('./AliRoot')])
def fake_exists(n):
return {"/alidist/aliroot.sh": True}
with patch.object(os.path, "exists", fake_exists):
doInit(args)
self.assertEqual(mock_git.mock_calls, CLONE_EVERYTHING)
mock_path.exists.assert_has_calls([call('.'), call('/sw/MIRROR'), call('/alidist'), call('./AliRoot')])

# Force fetch repos
mock_git.reset_mock()
mock_path.reset_mock()
args.fetchRepos = True
doInit(args)
self.assertEqual(mock_git.mock_calls, CLONE_EVERYTHING)
mock_path.exists.assert_has_calls([call('.'), call('/sw/MIRROR'), call('/alidist'), call('./AliRoot')])
# Force fetch repos
mock_git.reset_mock()
mock_path.reset_mock()
args.fetchRepos = True
doInit(args)
self.assertEqual(mock_git.mock_calls, CLONE_EVERYTHING)
mock_path.exists.assert_has_calls([call('.'), call('/sw/MIRROR'), call('/alidist'), call('./AliRoot')])


if __name__ == '__main__':
Expand Down
103 changes: 62 additions & 41 deletions tests/test_packagelist.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from textwrap import dedent
import unittest
from unittest import mock
from unittest.mock import patch
import os.path

from alibuild_helpers.cmd import getstatusoutput
from alibuild_helpers.utilities import getPackageList
Expand Down Expand Up @@ -71,14 +73,15 @@


class MockReader:
def __init__(self, url, dist=None):
def __init__(self, url: str, dist=None):
self._contents = RECIPES[url]
self.url = "mock://" + url

def __call__(self):
return self._contents



def getPackageListWithDefaults(packages, force_rebuild=()):
specs = {} # getPackageList will mutate this
return_values = getPackageList(
Expand Down Expand Up @@ -115,58 +118,70 @@ def test_disable(self):
This is was the only available behaviour in previous aliBuild versions
and must be preserved for backward compatibility.
"""
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["disable"])
self.assertIn("disable", systemPkgs)
self.assertNotIn("disable", ownPkgs)
self.assertNotIn("disable", specs)
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["disable"])
self.assertIn("disable", systemPkgs)
self.assertNotIn("disable", ownPkgs)
self.assertNotIn("disable", specs)

def test_replacement_given(self):
"""Check that specifying a replacement spec means it is used.
This also checks that if no recipe is given, we report the package as
a system package to the user.
"""
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["with-replacement"])
self.assertIn("with-replacement", specs)
self.assertEqual(specs["with-replacement"]["env"]["SENTINEL_VAR"], "magic")
# Make sure nothing is run by default.
self.assertEqual(specs["with-replacement"]["recipe"], "")
# If the replacement spec has no recipe, report to the user that we're
# taking the package from the system.
self.assertIn("with-replacement", systemPkgs)
self.assertNotIn("with-replacement", ownPkgs)
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["with-replacement"])
self.assertIn("with-replacement", specs)
self.assertEqual(specs["with-replacement"]["env"]["SENTINEL_VAR"], "magic")
# Make sure nothing is run by default.
self.assertEqual(specs["with-replacement"]["recipe"], "")
# If the replacement spec has no recipe, report to the user that we're
# taking the package from the system.
self.assertIn("with-replacement", systemPkgs)
self.assertNotIn("with-replacement", ownPkgs)

def test_replacement_recipe_given(self):
"""Check that specifying a replacement recipe means it is used.
Also check that we report to the user that a package will be compiled
when a replacement recipe is given.
"""
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["with-replacement-recipe"])
self.assertIn("with-replacement-recipe", specs)
self.assertIn("recipe", specs["with-replacement-recipe"])
self.assertEqual("true", specs["with-replacement-recipe"]["recipe"])
# If the replacement spec has a recipe, report to the user that we're
# compiling the package.
self.assertNotIn("with-replacement-recipe", systemPkgs)
self.assertIn("with-replacement-recipe", ownPkgs)
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["with-replacement-recipe"])
self.assertIn("with-replacement-recipe", specs)
self.assertIn("recipe", specs["with-replacement-recipe"])
self.assertEqual("true", specs["with-replacement-recipe"]["recipe"])
# If the replacement spec has a recipe, report to the user that we're
# compiling the package.
self.assertNotIn("with-replacement-recipe", systemPkgs)
self.assertIn("with-replacement-recipe", ownPkgs)

@mock.patch("alibuild_helpers.utilities.warning")
def test_missing_replacement_spec(self, mock_warning):
"""Check a warning is displayed when the replacement spec is not found."""
warning_msg = "falling back to building the package ourselves"
warning_exists = False
def side_effect(msg, *args, **kwargs):
nonlocal warning_exists
if warning_msg in str(msg):
warning_exists = True
mock_warning.side_effect = side_effect
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["missing-spec"])
self.assertTrue(warning_exists)
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
def side_effect(msg, *args, **kwargs):
nonlocal warning_exists
if warning_msg in str(msg):
warning_exists = True
mock_warning.side_effect = side_effect
specs, systemPkgs, ownPkgs, failedReqs, validDefaults = \
getPackageListWithDefaults(["missing-spec"])
self.assertTrue(warning_exists)



Expand All @@ -176,17 +191,23 @@ class ForceRebuildTestCase(unittest.TestCase):

def test_force_rebuild_recipe(self):
"""If the recipe specifies force_rebuild, it must be applied."""
specs, _, _, _, _ = getPackageListWithDefaults(["force-rebuild"])
self.assertTrue(specs["force-rebuild"]["force_rebuild"])
self.assertFalse(specs["defaults-release"]["force_rebuild"])
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
specs, _, _, _, _ = getPackageListWithDefaults(["force-rebuild"])
self.assertTrue(specs["force-rebuild"]["force_rebuild"])
self.assertFalse(specs["defaults-release"]["force_rebuild"])

def test_force_rebuild_command_line(self):
"""The --force-rebuild option must take precedence, if given."""
specs, _, _, _, _ = getPackageListWithDefaults(
["force-rebuild"], force_rebuild=["defaults-release", "force-rebuild"],
)
self.assertTrue(specs["force-rebuild"]["force_rebuild"])
self.assertTrue(specs["defaults-release"]["force_rebuild"])
def fake_exists(n):
return n in RECIPES.keys()
with patch.object(os.path, "exists", fake_exists):
specs, _, _, _, _ = getPackageListWithDefaults(
["force-rebuild"], force_rebuild=["defaults-release", "force-rebuild"],
)
self.assertTrue(specs["force-rebuild"]["force_rebuild"])
self.assertTrue(specs["defaults-release"]["force_rebuild"])


if __name__ == '__main__':
Expand Down
Loading

0 comments on commit f150599

Please sign in to comment.