From 75a4397a00e030d1c06ae99de1b3f387ade1e4a8 Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Tue, 5 Mar 2024 17:39:37 +0100 Subject: [PATCH] Clone SOURCES repos outside container This lets us always use speed-up options like --filter, even when we're building inside a container. We can also easily start using sapling for this in future by implementing the new SCM methods. This also helps with some annoying authentication-related issues, if running inside --docker, such as in CI. Fixes . When this is merged, we can also get rid of the `GIT_*` environment variables used in CI, and stop installing an especially-new Git version in our containers (since that was needed to understand the `GIT_*` variables). --- alibuild_helpers/build.py | 40 +++++-------------- alibuild_helpers/build_template.sh | 47 ++-------------------- alibuild_helpers/git.py | 30 ++++++++++++-- alibuild_helpers/init.py | 1 + alibuild_helpers/scm.py | 10 ++++- alibuild_helpers/sl.py | 9 +++++ alibuild_helpers/workarea.py | 57 ++++++++++++++++++++++++-- tests/test_build.py | 64 ++++++++++++++++++++++-------- tests/test_workarea.py | 2 +- 9 files changed, 162 insertions(+), 98 deletions(-) diff --git a/alibuild_helpers/build.py b/alibuild_helpers/build.py index 3ed4bfe1..576447ec 100644 --- a/alibuild_helpers/build.py +++ b/alibuild_helpers/build.py @@ -13,12 +13,12 @@ from alibuild_helpers.utilities import Hasher from alibuild_helpers.utilities import yamlDump from alibuild_helpers.utilities import resolve_tag, resolve_version -from alibuild_helpers.git import git, clone_speedup_options, Git +from alibuild_helpers.git import Git, git from alibuild_helpers.sl import Sapling from alibuild_helpers.scm import SCMError from alibuild_helpers.sync import remote_from_url import yaml -from alibuild_helpers.workarea import logged_scm, updateReferenceRepoSpec +from alibuild_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources from alibuild_helpers.log import ProgressPrint, log_current_package from glob import glob from textwrap import dedent @@ -69,9 +69,7 @@ def update_repo(package, git_prompt): if exists(os.path.join(specs[package]["source"], ".sl")): specs[package]["scm"] = Sapling() updateReferenceRepoSpec(args.referenceSources, package, specs[package], - fetch=args.fetchRepos, - usePartialClone=not args.docker, - allowGitPrompt=git_prompt) + fetch=args.fetchRepos, allowGitPrompt=git_prompt) # Retrieve git heads output = logged_scm(specs[package]["scm"], package, args.referenceSources, @@ -967,10 +965,6 @@ def doBuild(args, parser): if spec["cachedTarball"] else "No cache tarballs found") # The actual build script. - referenceStatement = "" - if "reference" in spec: - referenceStatement = "export GIT_REFERENCE=${GIT_REFERENCE_OVERRIDE:-%s}/%s" % (dirname(spec["reference"]), basename(spec["reference"])) - debug("spec = %r", spec) cmd_raw = "" @@ -982,22 +976,14 @@ def doBuild(args, parser): from pkg_resources import resource_string cmd_raw = resource_string("alibuild_helpers", 'build_template.sh') - source = spec.get("source", "") - # Shortend the commit hash in case it's a real commit hash and not simply - # the tag. - commit_hash = spec["commit_hash"] - if spec["tag"] != spec["commit_hash"]: - commit_hash = spec["commit_hash"][0:10] - - # Split the source in two parts, sourceDir and sourceName. This is done so - # that when we use Docker we can replace sourceDir with the correct - # container path, if required. No changes for what concerns the standard - # bash builds, though. if args.docker: cachedTarball = re.sub("^" + workDir, "/sw", spec["cachedTarball"]) else: cachedTarball = spec["cachedTarball"] + if not cachedTarball: + checkout_sources(spec, workDir, args.referenceSources, args.docker) + scriptDir = join(workDir, "SPECS", args.architecture, spec["package"], spec["version"] + "-" + spec["revision"]) @@ -1011,11 +997,6 @@ def doBuild(args, parser): "workDir": workDir, "configDir": abspath(args.configDir), "incremental_recipe": spec.get("incremental_recipe", ":"), - "sourceDir": (dirname(source) + "/") if source else "", - "sourceName": basename(source) if source else "", - "referenceStatement": referenceStatement, - "gitOptionsStatement": "" if args.docker else - "export GIT_CLONE_SPEEDUP=" + quote(" ".join(clone_speedup_options())), "requires": " ".join(spec["requires"]), "build_requires": " ".join(spec["build_requires"]), "runtime_requires": " ".join(spec["runtime_requires"]), @@ -1028,14 +1009,14 @@ def doBuild(args, parser): ("BUILD_REQUIRES", " ".join(spec["build_requires"])), ("CACHED_TARBALL", cachedTarball), ("CAN_DELETE", args.aggressiveCleanup and "1" or ""), - ("COMMIT_HASH", commit_hash), + # Shorten the commit hash if it's a real commit hash and not simply the tag. + ("COMMIT_HASH", spec["tag"] if spec["tag"] == spec["commit_hash"] else spec["commit_hash"][:10]), ("DEPS_HASH", spec.get("deps_hash", "")), ("DEVEL_HASH", spec.get("devel_hash", "")), ("DEVEL_PREFIX", develPrefix), ("BUILD_FAMILY", spec["build_family"]), ("GIT_COMMITTER_NAME", "unknown"), ("GIT_COMMITTER_EMAIL", "unknown"), - ("GIT_TAG", spec["tag"]), ("INCREMENTAL_BUILD_HASH", spec.get("incremental_hash", "0")), ("JOBS", str(args.jobs)), ("PKGHASH", spec["hash"]), @@ -1048,7 +1029,6 @@ def doBuild(args, parser): ("FULL_RUNTIME_REQUIRES", " ".join(spec["full_runtime_requires"])), ("FULL_BUILD_REQUIRES", " ".join(spec["full_build_requires"])), ("FULL_REQUIRES", " ".join(spec["full_requires"])), - ("WRITE_REPO", spec.get("write_repo", source)), ] # Add the extra environment as passed from the command line. buildEnvironment += [e.partition('=')[::2] for e in args.environment] @@ -1059,15 +1039,13 @@ def doBuild(args, parser): build_command = ( "docker run --rm --entrypoint= --user $(id -u):$(id -g) " "-v {workdir}:/sw -v {scriptDir}/build.sh:/build.sh:ro " - "-e GIT_REFERENCE_OVERRIDE=/mirror -e WORK_DIR_OVERRIDE=/sw " "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} " - "{overrideSource} {extraArgs} {image} bash -ex /build.sh" + "-e WORK_DIR_OVERRIDE=/sw {extraArgs} {image} bash -ex /build.sh" ).format( image=quote(args.dockerImage), workdir=quote(abspath(args.workDir)), scriptDir=quote(scriptDir), extraArgs=" ".join(map(quote, args.docker_extra_args)), - overrideSource="-e SOURCE0_DIR_OVERRIDE=/" if source.startswith("/") else "", additionalEnv=" ".join( "-e {}={}".format(var, quote(value)) for var, value in buildEnvironment), # Used e.g. by O2DPG-sim-tests to find the O2DPG repository. diff --git a/alibuild_helpers/build_template.sh b/alibuild_helpers/build_template.sh index 68d6a647..82f81e79 100644 --- a/alibuild_helpers/build_template.sh +++ b/alibuild_helpers/build_template.sh @@ -27,7 +27,6 @@ export PATH=$WORK_DIR/wrapper-scripts:$PATH # - DEPS_HASH # - DEVEL_HASH # - DEVEL_PREFIX -# - GIT_TAG # - INCREMENTAL_BUILD_HASH # - JOBS # - PKGHASH @@ -36,22 +35,20 @@ export PATH=$WORK_DIR/wrapper-scripts:$PATH # - PKGVERSION # - REQUIRES # - RUNTIME_REQUIRES -# - WRITE_REPO export PKG_NAME="$PKGNAME" export PKG_VERSION="$PKGVERSION" export PKG_BUILDNUM="$PKGREVISION" -export SOURCE0="${SOURCE0_DIR_OVERRIDE:-%(sourceDir)s}%(sourceName)s" export PKGPATH=${ARCHITECTURE}/${PKGNAME}/${PKGVERSION}-${PKGREVISION} mkdir -p "$WORK_DIR/BUILD" "$WORK_DIR/SOURCES" "$WORK_DIR/TARS" \ "$WORK_DIR/SPECS" "$WORK_DIR/INSTALLROOT" export BUILDROOT="$WORK_DIR/BUILD/$PKGHASH" -# In case the repository is local, it means we are in development mode, so we -# install directly in $WORK_DIR/$PKGPATH so that we can do make install -# directly into BUILD/$PKGPATH and have changes being propagated. -if [ "${SOURCE0:0:1}" == "/" ]; then +# If we are in development mode, then install directly in $WORK_DIR/$PKGPATH, +# so that we can do "make install" directly into BUILD/$PKGPATH and have +# changes being propagated. +if [ -n "$DEVEL_HASH" ]; then export INSTALLROOT="$WORK_DIR/$PKGPATH" else export INSTALLROOT="$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH" @@ -59,13 +56,6 @@ fi export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/$COMMIT_HASH" export BUILDDIR="$BUILDROOT/$PKGNAME" -SHORT_TAG=${GIT_TAG:0:10} -mkdir -p $(dirname $SOURCEDIR) -if [[ ${COMMIT_HASH} != ${GIT_TAG} && "${SHORT_TAG:-0}" != ${COMMIT_HASH} ]]; then - GIT_TAG_DIR=${GIT_TAG:-0} - GIT_TAG_DIR=${GIT_TAG_DIR//\//_} - ln -snf ${COMMIT_HASH} "$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/${GIT_TAG_DIR}" -fi rm -fr "$WORK_DIR/INSTALLROOT/$PKGHASH" # We remove the build directory only if we are not in incremental mode. if [[ "$INCREMENTAL_BUILD_HASH" == 0 ]] && ! rm -rf "$BUILDROOT"; then @@ -114,35 +104,6 @@ if [[ $DEVEL_PREFIX ]]; then ln -snf $PKGHASH $WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX fi -# Reference statements -%(referenceStatement)s -%(gitOptionsStatement)s - -if [ -z "$CACHED_TARBALL" ]; then - case "$SOURCE0" in - '') # SOURCE0 is empty, so just create an empty SOURCEDIR. - mkdir -p "$SOURCEDIR" ;; - /*) # SOURCE0 is an absolute path, so just make a symlink there. - ln -snf "$SOURCE0" "$SOURCEDIR" ;; - *) # SOURCE0 is a relative path or URL, so clone/checkout the git repo from there. - if cd "$SOURCEDIR" 2>/dev/null; then - # Folder is already present, but check that it is the right tag - if ! git checkout -f "$GIT_TAG"; then - # If we can't find the tag, it might be new. Fetch tags and try again. - git fetch -f "$SOURCE0" "refs/tags/$GIT_TAG:refs/tags/$GIT_TAG" - git checkout -f "$GIT_TAG" - fi - else - # In case there is a stale link / file, for whatever reason. - rm -rf "$SOURCEDIR" - git clone -n $GIT_CLONE_SPEEDUP ${GIT_REFERENCE:+--reference "$GIT_REFERENCE"} "$SOURCE0" "$SOURCEDIR" - cd "$SOURCEDIR" - git remote set-url --push origin "$WRITE_REPO" - git checkout -f "$GIT_TAG" - fi ;; - esac -fi - cd "$BUILDDIR" # Actual build script, as defined in the recipe diff --git a/alibuild_helpers/git.py b/alibuild_helpers/git.py index 9ce232f8..8f0717fa 100644 --- a/alibuild_helpers/git.py +++ b/alibuild_helpers/git.py @@ -18,34 +18,58 @@ def clone_speedup_options(): class Git(SCM): name = "Git" + def checkedOutCommitName(self, directory): return git(("rev-parse", "HEAD"), directory) + def branchOrRef(self, directory): out = git(("rev-parse", "--abbrev-ref", "HEAD"), directory=directory) if out == "HEAD": out = git(("rev-parse", "HEAD"), directory)[:10] return out + def exec(self, *args, **kwargs): return git(*args, **kwargs) + def parseRefs(self, output): return { git_ref: git_hash for git_hash, sep, git_ref in (line.partition("\t") for line in output.splitlines()) if sep } + def listRefsCmd(self, repository): return ["ls-remote", "--heads", "--tags", repository] - def cloneCmd(self, source, referenceRepo, usePartialClone): + + def cloneReferenceCmd(self, source, referenceRepo, usePartialClone): cmd = ["clone", "--bare", source, referenceRepo] if usePartialClone: cmd.extend(clone_speedup_options()) return cmd - def fetchCmd(self, source): - return ["fetch", "-f", "--tags", source, "+refs/heads/*:refs/heads/*"] + + def cloneSourceCmd(self, source, destination, referenceRepo, usePartialClone): + cmd = ["clone", "-n", source, destination] + if referenceRepo: + cmd.extend(["--reference", referenceRepo]) + if usePartialClone: + cmd.extend(clone_speedup_options()) + return cmd + + def checkoutCmd(self, ref): + return ["checkout", "-f", ref] + + def fetchCmd(self, source, *refs): + return ["fetch", "-f", source, *refs] + + def setWriteUrlCmd(self, url): + return ["remote", "set-url", "--push", "origin", url] + def diffCmd(self, directory): return "cd %s && git diff -r HEAD && git status --porcelain" % directory + def checkUntracked(self, line): return line.startswith("?? ") + def git(args, directory=".", check=True, prompt=True): debug("Executing git %s (in directory %s)", " ".join(args), directory) # We can't use git --git-dir=%s/.git or git -C %s here as the former requires diff --git a/alibuild_helpers/init.py b/alibuild_helpers/init.py index d3bc896f..d9d3b04e 100644 --- a/alibuild_helpers/init.py +++ b/alibuild_helpers/init.py @@ -66,6 +66,7 @@ def doInit(args): for p in pkgs: spec = specs.get(p["name"]) + spec["is_devel_pkg"] = False spec["scm"] = Git() dieOnError(spec is None, "cannot find recipe for package %s" % p["name"]) dest = join(args.develPrefix, spec["package"]) diff --git a/alibuild_helpers/scm.py b/alibuild_helpers/scm.py index 985879df..ac90e460 100644 --- a/alibuild_helpers/scm.py +++ b/alibuild_helpers/scm.py @@ -15,7 +15,15 @@ def parseRefs(self, output): raise NotImplementedError def exec(self, *args, **kwargs): raise NotImplementedError - def cloneCmd(self, spec, referenceRepo, usePartialClone): + def checkoutCmd(self, tag): + raise NotImplementedError + def fetchCmd(self, remote, *refs): + raise NotImplementedError + def cloneReferenceCmd(self, spec, referenceRepo, usePartialClone): + raise NotImplementedError + def cloneSourceCmd(self, spec, referenceRepo, usePartialClone): + raise NotImplementedError + def setWriteUrlCmd(self, url): raise NotImplementedError def diffCmd(self, directory): raise NotImplementedError diff --git a/alibuild_helpers/sl.py b/alibuild_helpers/sl.py index 9f72db42..ea858d6c 100644 --- a/alibuild_helpers/sl.py +++ b/alibuild_helpers/sl.py @@ -6,6 +6,7 @@ SL_COMMAND_TIMEOUT_SEC = 120 """How many seconds to let any sl command execute before being terminated.""" + # Sapling is a novel SCM by Meta (i.e. Facebook) that is fully compatible with # git, but has a different command line interface. Among the reasons why it's # worth suporting it is the ability to handle unnamed branches, the ability to @@ -14,26 +15,34 @@ # command line from each commit of a branch. class Sapling(SCM): name = "Sapling" + def checkedOutCommitName(self, directory): return sapling(("whereami", ), directory) + def branchOrRef(self, directory): # Format is [+] identity = sapling(("identify", ), directory) return identity.split(" ")[-1] + def exec(self, *args, **kwargs): return sapling(*args, **kwargs) + def parseRefs(self, output): return { sl_ref: sl_hash for sl_ref, sep, sl_hash in (line.partition("\t") for line in output.splitlines()) if sep } + def listRefsCmd(self, repository): return ["bookmark", "--list", "--remote", "-R", repository] + def diffCmd(self, directory): return "cd %s && sl diff && sl status" % directory + def checkUntracked(self, line): return line.startswith("? ") + def sapling(args, directory=".", check=True, prompt=True): debug("Executing sl %s (in directory %s)", " ".join(args), directory) # We can't use git --git-dir=%s/.git or git -C %s here as the former requires diff --git a/alibuild_helpers/workarea.py b/alibuild_helpers/workarea.py index 907198e9..bfb6cde8 100644 --- a/alibuild_helpers/workarea.py +++ b/alibuild_helpers/workarea.py @@ -2,6 +2,7 @@ import errno import os import os.path +import shutil import tempfile try: from collections import OrderedDict @@ -9,7 +10,7 @@ from ordereddict import OrderedDict from alibuild_helpers.log import dieOnError, debug, error -from alibuild_helpers.utilities import call_ignoring_oserrors +from alibuild_helpers.utilities import call_ignoring_oserrors, symlink FETCH_LOG_NAME = "fetch-log.txt" @@ -112,10 +113,10 @@ def updateReferenceRepo(referenceSources, p, spec, return None # no reference can be found and created (not fatal) if not os.path.exists(referenceRepo): - cmd = scm.cloneCmd(spec["source"], referenceRepo, usePartialClone) + cmd = scm.cloneReferenceCmd(spec["source"], referenceRepo, usePartialClone) logged_scm(scm, p, referenceSources, cmd, ".", allowGitPrompt) elif fetch: - cmd = scm.fetchCmd(spec["source"]) + cmd = scm.fetchCmd(spec["source"], "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*") logged_scm(scm, p, referenceSources, cmd, referenceRepo, allowGitPrompt) return referenceRepo # reference is read-write @@ -127,3 +128,53 @@ def is_writeable(dirpath): return True except: return False + + +def checkout_sources(spec, work_dir, reference_sources, containerised_build): + """Check out sources to be compiled, potentially from a given reference.""" + scm = spec["scm"] + + def scm_exec(command, directory=".", check=True): + """Run the given SCM command, simulating a shell exit code.""" + try: + logged_scm(scm, spec["package"], reference_sources, command, directory, prompt=False) + except SystemExit as exc: + if check: + raise + return exc.code + return 0 + + source_parent_dir = os.path.join(work_dir, "SOURCES", spec["package"], spec["version"]) + source_dir = os.path.join(source_parent_dir, spec["commit_hash"]) + os.makedirs(source_parent_dir, exist_ok=True) + + if spec["commit_hash"] != spec["tag"]: + symlink(spec["commit_hash"], os.path.join(source_parent_dir, spec["tag"].replace("/", "_"))) + + if "source" not in spec: + # There are no sources, so just create an empty SOURCEDIR. + os.makedirs(source_dir, exist_ok=True) + elif spec["is_devel_pkg"]: + shutil.rmtree(source_dir, ignore_errors=True) + # In a container, we mount development packages' source dirs in /. + # Outside a container, we have access to the source dir directly. + symlink("/" + os.path.basename(spec["source"]) + if containerised_build else spec["source"], + source_dir) + elif os.path.isdir(source_dir): + # Sources are a relative path or URL and the local repo already exists, so + # checkout the right commit there. + err = scm_exec(scm.checkoutCmd(spec["tag"]), source_dir, check=False) + if err: + # If we can't find the tag, it might be new. Fetch tags and try again. + tag_ref = "refs/tags/{0}:refs/tags/{0}".format(spec["tag"]) + scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) + scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) + else: + # Sources are a relative path or URL and don't exist locally yet, so clone + # and checkout the git repo from there. + shutil.rmtree(source_dir, ignore_errors=True) + scm_exec(scm.cloneSourceCmd(spec["source"], source_dir, spec.get("reference"), + usePartialClone=True)) + scm_exec(scm.setWriteUrlCmd(spec.get("write_repo", spec["source"])), source_dir) + scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) diff --git a/tests/test_build.py b/tests/test_build.py index a912a188..e0184355 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -17,7 +17,6 @@ except ImportError: from ordereddict import OrderedDict -from alibuild_helpers.cmd import is_string from alibuild_helpers.utilities import parseRecipe, resolve_tag from alibuild_helpers.build import doBuild, storeHashes, generate_initdotsh @@ -102,10 +101,25 @@ "5afae57bfc6a374e74c1c4427698ab5edebce0bc") -GIT_CLONE_ZLIB_ARGS = ("clone", "--bare", "https://github.com/star-externals/zlib", - "/sw/MIRROR/zlib", "--filter=blob:none"), ".", False -GIT_FETCH_ROOT_ARGS = ("fetch", "-f", "--tags", "https://github.com/root-mirror/root", - "+refs/heads/*:refs/heads/*"), "/sw/MIRROR/root", False +GIT_CLONE_REF_ZLIB_ARGS = ("clone", "--bare", "https://github.com/star-externals/zlib", + "/sw/MIRROR/zlib", "--filter=blob:none"), ".", False +GIT_CLONE_SRC_ZLIB_ARGS = ("clone", "-n", "https://github.com/star-externals/zlib", + "/sw/SOURCES/zlib/v1.2.3/8822efa61f2a385e0bc83ca5819d608111b2168a", + "--reference", "/sw/MIRROR/zlib", "--filter=blob:none"), ".", False +GIT_SET_URL_ZLIB_ARGS = ("remote", "set-url", "--push", "origin", "https://github.com/star-externals/zlib"), \ + "/sw/SOURCES/zlib/v1.2.3/8822efa61f2a385e0bc83ca5819d608111b2168a", False +GIT_CHECKOUT_ZLIB_ARGS = ("checkout", "-f", "master"), \ + "/sw/SOURCES/zlib/v1.2.3/8822efa61f2a385e0bc83ca5819d608111b2168a", False + +GIT_FETCH_REF_ROOT_ARGS = ("fetch", "-f", "https://github.com/root-mirror/root", "+refs/tags/*:refs/tags/*", + "+refs/heads/*:refs/heads/*"), "/sw/MIRROR/root", False +GIT_CLONE_SRC_ROOT_ARGS = ("clone", "-n", "https://github.com/root-mirror/root", + "/sw/SOURCES/ROOT/v6-08-30/f7b336611753f1f4aaa94222b0d620748ae230c0", + "--reference", "/sw/MIRROR/root", "--filter=blob:none"), ".", False +GIT_SET_URL_ROOT_ARGS = ("remote", "set-url", "--push", "origin", "https://github.com/root-mirror/root"), \ + "/sw/SOURCES/ROOT/v6-08-30/f7b336611753f1f4aaa94222b0d620748ae230c0", False +GIT_CHECKOUT_ROOT_ARGS = ("checkout", "-f", "v6-08-00-patches"), \ + "/sw/SOURCES/ROOT/v6-08-30/f7b336611753f1f4aaa94222b0d620748ae230c0", False def dummy_git(args, directory=".", check=True, prompt=True): @@ -114,8 +128,14 @@ def dummy_git(args, directory=".", check=True, prompt=True): (("rev-parse", "HEAD"), "/alidist", True): "6cec7b7b3769826219dfa85e5daa6de6522229a0", (("ls-remote", "--heads", "--tags", "/sw/MIRROR/root"), ".", False): (0, TEST_ROOT_GIT_REFS), (("ls-remote", "--heads", "--tags", "/sw/MIRROR/zlib"), ".", False): (0, TEST_ZLIB_GIT_REFS), - GIT_CLONE_ZLIB_ARGS: (0, ""), - GIT_FETCH_ROOT_ARGS: (0, ""), + GIT_CLONE_REF_ZLIB_ARGS: (0, ""), + GIT_CLONE_SRC_ZLIB_ARGS: (0, ""), + GIT_SET_URL_ZLIB_ARGS: (0, ""), + GIT_CHECKOUT_ZLIB_ARGS: (0, ""), + GIT_FETCH_REF_ROOT_ARGS: (0, ""), + GIT_CLONE_SRC_ROOT_ARGS: (0, ""), + GIT_SET_URL_ROOT_ARGS: (0, ""), + GIT_CHECKOUT_ROOT_ARGS: (0, ""), }[(tuple(args), directory, check)] @@ -182,8 +202,6 @@ def dummy_exists(x): # A few errors we should handle, together with the expected result -@patch("alibuild_helpers.build.clone_speedup_options", - new=MagicMock(return_value=["--filter=blob:none"])) @patch("alibuild_helpers.git.clone_speedup_options", new=MagicMock(return_value=["--filter=blob:none"])) @patch("alibuild_helpers.build.BASH", new="/bin/bash") @@ -200,8 +218,11 @@ class BuildTestCase(unittest.TestCase): @patch("alibuild_helpers.utilities.warning") @patch("alibuild_helpers.build.readDefaults", new=MagicMock(return_value=(OrderedDict({"package": "defaults-release", "disable": []}), ""))) + @patch("shutil.rmtree", new=MagicMock(return_value=None)) + @patch("os.makedirs", new=MagicMock(return_value=None)) @patch("alibuild_helpers.build.makedirs", new=MagicMock(return_value=None)) @patch("alibuild_helpers.build.symlink", new=MagicMock(return_value=None)) + @patch("alibuild_helpers.workarea.symlink", new=MagicMock(return_value=None)) @patch("alibuild_helpers.utilities.open", new=lambda x: { "/alidist/root.sh": StringIO(TEST_ROOT_RECIPE), "/alidist/zlib.sh": StringIO(TEST_ZLIB_RECIPE), @@ -270,11 +291,13 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_sys, mo ) mock_sys.version_info = sys.version_info - clone_args, clone_dir, clone_check = GIT_CLONE_ZLIB_ARGS - fetch_args, fetch_dir, fetch_check = GIT_FETCH_ROOT_ARGS + def mkcall(args): + cmd, directory, check = args + return call(list(cmd), directory=directory, check=check, prompt=False) + common_calls = [ call(("rev-parse", "HEAD"), args.configDir), - call(list(clone_args), directory=clone_dir, check=clone_check, prompt=False), + mkcall(GIT_CLONE_REF_ZLIB_ARGS), call(["ls-remote", "--heads", "--tags", args.referenceSources + "/zlib"], directory=".", check=False, prompt=False), call(["ls-remote", "--heads", "--tags", args.referenceSources + "/root"], @@ -287,8 +310,17 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_sys, mo doBuild(args, mock_parser) mock_warning.assert_called_with("%s.sh contains a recipe, which will be ignored", "defaults-release") mock_debug.assert_called_with("Everything done") - self.assertEqual(mock_git_git.call_count, len(common_calls)) - mock_git_git.assert_has_calls(common_calls, any_order=True) + # After this run, .build-hash files will be simulated to exist + # already, so sw/SOURCES repos must only be checked out on this run. + mock_git_git.assert_has_calls(common_calls + [ + mkcall(GIT_CLONE_SRC_ZLIB_ARGS), + mkcall(GIT_SET_URL_ZLIB_ARGS), + mkcall(GIT_CHECKOUT_ZLIB_ARGS), + mkcall(GIT_CLONE_SRC_ROOT_ARGS), + mkcall(GIT_SET_URL_ROOT_ARGS), + mkcall(GIT_CHECKOUT_ROOT_ARGS), + ], any_order=True) + self.assertEqual(mock_git_git.call_count, len(common_calls) + 6) # Force fetching repos mock_git_git.reset_mock() @@ -301,10 +333,10 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_sys, mo mock_listdir.assert_called_with("/sw/TARS/osx_x86-64/ROOT") # We can't compare directly against the list of calls here as they # might happen in any order. - self.assertEqual(mock_git_git.call_count, len(common_calls) + 1) mock_git_git.assert_has_calls(common_calls + [ - call(list(fetch_args), directory=fetch_dir, check=fetch_check, prompt=False), + mkcall(GIT_FETCH_REF_ROOT_ARGS), ], any_order=True) + self.assertEqual(mock_git_git.call_count, len(common_calls) + 1) def setup_spec(self, script): """Parse the alidist recipe in SCRIPT and return its spec.""" diff --git a/tests/test_workarea.py b/tests/test_workarea.py index a43d5975..b127994a 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -63,7 +63,7 @@ def test_reference_sources_updated(self, mock_git, mock_open, mock_makedirs, moc mock_exists.assert_has_calls([]) mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True) mock_git.assert_called_once_with([ - "fetch", "-f", "--tags", spec["source"], "+refs/heads/*:refs/heads/*", + "fetch", "-f", spec["source"], "+refs/tags/*:refs/tags/*", "+refs/heads/*:refs/heads/*", ], directory="%s/sw/MIRROR/aliroot" % getcwd(), check=False, prompt=True) self.assertEqual(spec.get("reference"), "%s/sw/MIRROR/aliroot" % getcwd())