Merge pull request #546 from truenas/NAS-125472

NAS-125472 / 24.04 / Validate cached bootstrap directory based off on the reference group/passwd files

scale_build/bootstrap/cache.py

@@ -1,6 +1,8 @@
 import os
+import shutil
 
 from scale_build.utils.paths import CACHE_DIR
+from scale_build.utils.reference_files import compare_reference_files
 from scale_build.utils.run import run
 
 from .hash import get_all_repo_hash
@@ -52,6 +54,16 @@ def mirror_cache_intact(self):
             self.logger.debug('Upstream repo changed! Removing squashfs cache to re-create.')
             intact = False
 
+        if intact:
+            self.restore_cache(self.chroot_basedir)
+            for _, diff in compare_reference_files(cut_nonexistent_user_group_membership=True):
+                if diff:
+                    intact = False
+                    break
+
+            # Remove the temporary restored cached directory
+            shutil.rmtree(self.chroot_basedir, ignore_errors=True)
+
         if not intact:
             self.remove_cache()
 

scale_build/update_image.py

@@ -1,4 +1,3 @@
-import difflib
 import logging
 import os
 
@@ -11,33 +10,10 @@
 from .image.update import install_rootfs_packages, build_rootfs_image
 from .utils.logger import LoggingContext
 from .utils.paths import CHROOT_BASEDIR, LOG_DIR, REFERENCE_FILES, REFERENCE_FILES_DIR, RELEASE_DIR
+from .utils.reference_files import compare_reference_files
 
-logger = logging.getLogger(__name__)
-
-
-def compare_reference_files(cut_nonexistent_user_group_membership=False):
-    for reference_file in REFERENCE_FILES:
-        with open(os.path.join(REFERENCE_FILES_DIR, reference_file)) as f:
-            reference = f.readlines()
-
-        if cut_nonexistent_user_group_membership:
-            if reference_file == 'etc/group':
-                # `etc/group` on newly installed system can't have group membership information for users that have
-                # not been created yet.
-                with open(os.path.join(CHROOT_BASEDIR, 'etc/passwd')) as f:
-                    reference_users = {line.split(':')[0] for line in f.readlines()}
 
-                for i, line in enumerate(reference):
-                    bits = line.rstrip().split(':')
-                    bits[3] = ','.join([user for user in bits[3].split(',') if user in reference_users])
-                    reference[i] = ':'.join(bits) + '\n'
-
-        with open(os.path.join(CHROOT_BASEDIR, reference_file)) as f:
-            real = f.readlines()
-
-        diff = list(difflib.unified_diff(reference, real))
-
-        yield reference_file, diff[3:]
+logger = logging.getLogger(__name__)
 
 
 def build_update_image():

scale_build/utils/reference_files.py

@@ -0,0 +1,34 @@
+import os
+import difflib
+
+from scale_build.exceptions import CallError
+
+from .paths import REFERENCE_FILES_DIR, REFERENCE_FILES, CHROOT_BASEDIR
+
+
+def compare_reference_files(cut_nonexistent_user_group_membership=False):
+    for reference_file in REFERENCE_FILES:
+        with open(os.path.join(REFERENCE_FILES_DIR, reference_file)) as f:
+            reference = f.readlines()
+
+        if not os.path.exists(os.path.join(CHROOT_BASEDIR, reference_file)):
+            raise CallError(f'File {reference_file!r} does not exist in cached chroot')
+
+        if cut_nonexistent_user_group_membership:
+            if reference_file == 'etc/group':
+                # `etc/group` on newly installed system can't have group membership information for users that have
+                # not been created yet.
+                with open(os.path.join(CHROOT_BASEDIR, 'etc/passwd')) as f:
+                    reference_users = {line.split(':')[0] for line in f.readlines()}
+
+                for i, line in enumerate(reference):
+                    bits = line.rstrip().split(':')
+                    bits[3] = ','.join([user for user in bits[3].split(',') if user in reference_users])
+                    reference[i] = ':'.join(bits) + '\n'
+
+        with open(os.path.join(CHROOT_BASEDIR, reference_file)) as f:
+            real = f.readlines()
+
+        diff = list(difflib.unified_diff(reference, real))
+
+        yield reference_file, diff[3:]

scale_build/validate.py

@@ -3,9 +3,11 @@
 import logging
 import shutil
 
+from truenas_install import fhs
+
 from .exceptions import CallError, MissingPackagesException
 from .utils.manifest import validate_manifest
-from truenas_install import fhs
+from .utils.paths import REFERENCE_FILES, REFERENCE_FILES_DIR
 
 
 logger = logging.getLogger(__name__)
@@ -32,6 +34,11 @@ def validate_system_state():
     if missing_packages:
         raise MissingPackagesException(missing_packages)
 
+    if missing_files := [
+        f for f in map(lambda f: os.path.join(REFERENCE_FILES_DIR, f), REFERENCE_FILES) if not os.path.exists(f)
+    ]:
+        raise CallError(f'Missing reference files: {", ".join(missing_files)!r}')
+
 
 def validate_datasets():
     try: