buildextend-live: Add support OSBUILD

src/cmd-buildextend-live

@@ -23,6 +23,14 @@ from cosalib.cmdlib import runcmd, sha256sum_file
 from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob
 from cosalib.meta import GenericBuildMeta
 
+# Size of file used to embed an Ignition config within a CPIO.
+IGNITION_IMG_SIZE = 256 * 1024
+
+# Size of the file used to embed miniso data.
+MINISO_DATA_FILE_SIZE = 16 * 1024
+
+COSA_OSBUILD_LIVEISO = os.getenv("COSA_OSBUILD_LIVEISO", "")
+
 live_exclude_kargs = set([
     '$ignition_firstboot',   # unsubstituted variable in grub config
     'console',               # no serial console by default on ISO
@@ -91,32 +99,39 @@ name_version = f'{base_name}-{args.build}'
 # to shorten this more intelligently, otherwise we truncate the
 # version which may impede uniqueness.
 volid = name_version[0:32]
+build_path = os.path.abspath(f"{builddir}/{base_name}-{args.build}")
+kernel_name = f'{base_name}-{args.build}-live-kernel-{basearch}'
+initramfs_name = f'{base_name}-{args.build}-live-initramfs.{basearch}.img'
+rootfs_name = f'{base_name}-{args.build}-live-rootfs.{basearch}.img'
+kernel_file = os.path.join(builddir, kernel_name)
+initramfs_file = os.path.join(builddir, initramfs_name)
+rootfs_file = os.path.join(builddir, rootfs_name)
+
+if COSA_OSBUILD_LIVEISO != "":
+    data = {
+        "buildid": args.build,
+        "imgid": iso_name,
+        "ostree-commit": buildmeta_commit,
+        "container-imgref": "",
+        "deploy-via-container": "",
+        "osname": base_name,
+        "ostree-ref": args.build,
+        "ostree-container": f"{build_path}-ostree.{basearch}.ociarchive",
+        "metal-filename": f"{build_path}-metal.{basearch}.raw",
+        "metal4k-filename": f"{build_path}-metal4k.{basearch}.raw",
+        "ostree-repo": repo,
+        "extra-kargs-string": "mitigations=auto,nosmt",
+        "image-type": "live-iso",
+        "cloud-image-size": "10240",
+        "metal-image-size": "2405",
+        "squashfs-compression": squashfs_compression,
+        "rootfs-size": 0,
+        "live-efiboot-img-size": 16
+    }
 
-tmpdir = os.environ.get("FORCE_TMPDIR", f"{workdir}/tmp/buildpost-live")
-if os.path.isdir(tmpdir):
-    shutil.rmtree(tmpdir)
-
-tmpisoroot = os.path.join(tmpdir, 'live')
-tmpisocoreos = os.path.join(tmpisoroot, 'coreos')
-tmpisoimages = os.path.join(tmpisoroot, 'images')
-tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot')
-tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
-# contents of initramfs on both PXE and ISO
-tmpinitrd_base = os.path.join(tmpdir, 'initrd')
-# contents of rootfs image
-tmpinitrd_rootfs = os.path.join(tmpdir, 'initrd-rootfs')
-
-for d in (tmpdir, tmpisoroot, tmpisocoreos, tmpisoimages, tmpisoimagespxe,
-        tmpisoisolinux, tmpinitrd_base, tmpinitrd_rootfs):
-    os.mkdir(d)
-
-# Size of file used to embed an Ignition config within a CPIO.
-ignition_img_size = 256 * 1024
-
-# Size of the file used to embed miniso data.
-miniso_data_file_size = 16 * 1024
-
-
+    image_for_disk_json = "runvm.json"
+    with open(image_for_disk_json, 'w') as file:
+        json.dump(data, file, indent=4)
 # The kernel requires that uncompressed cpio archives appended to an initrd
 # start on a 4-byte boundary.  If there's misalignment, it stops unpacking
 # and says:
@@ -133,6 +148,35 @@ def align_initrd_for_uncompressed_append(destf):
         destf.write(b'\0' * (4 - offset % 4))
 
 
+def update_buildmeta():
+    buildmeta['images'].update({
+        'live-iso': {
+            'path': iso_name,
+            'sha256': sha256sum_file(tmpisofile),
+            'skip-compression': True,
+        }
+    })
+    buildmeta['images'].update({
+        'live-kernel': {
+            'path': kernel_name,
+            'sha256': sha256sum_file(kernel_file),
+            'skip-compression': True,
+        },
+        'live-initramfs': {
+            'path': initramfs_name,
+            'sha256': sha256sum_file(initramfs_file),
+            'skip-compression': True,
+        },
+        'live-rootfs': {
+            'path': rootfs_name,
+            'sha256': sha256sum_file(rootfs_file),
+            'skip-compression': True,
+        }
+    })
+    buildmeta.write(artifact_name='live')
+    print(f"Updated: {buildmeta_path}")
+
+
 # Return OS features table for features.json, which is read by
 # coreos-installer {iso|pxe} customize
 def get_os_features():
@@ -222,6 +266,24 @@ def generate_iso():
     kargs_file = 'kargs.json'
     igninfo_file = 'igninfo.json'
 
+    tmpdir = os.environ.get("FORCE_TMPDIR", f"{workdir}/tmp/buildpost-live")
+    if os.path.isdir(tmpdir):
+        shutil.rmtree(tmpdir)
+
+    tmpisoroot = os.path.join(tmpdir, 'live')
+    tmpisocoreos = os.path.join(tmpisoroot, 'coreos')
+    tmpisoimages = os.path.join(tmpisoroot, 'images')
+    tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot')
+    tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
+    # contents of initramfs on both PXE and ISO
+    tmpinitrd_base = os.path.join(tmpdir, 'initrd')
+    # contents of rootfs image
+    tmpinitrd_rootfs = os.path.join(tmpdir, 'initrd-rootfs')
+
+    for d in (tmpdir, tmpisoroot, tmpisocoreos, tmpisoimages, tmpisoimagespxe,
+            tmpisoisolinux, tmpinitrd_base, tmpinitrd_rootfs):
+        os.mkdir(d)
+
     tmpisofile = os.path.join(tmpdir, iso_name)
 
     img_metal_obj = buildmeta.get_artifact_meta("metal", unmerged=True)["images"].get("metal")
@@ -289,7 +351,7 @@ def generate_iso():
     # handled lower down
     if basearch != 's390x':
         with open(os.path.join(tmpisoimages, 'ignition.img'), 'wb') as fdst:
-            fdst.write(bytes(ignition_img_size))
+            fdst.write(bytes(IGNITION_IMG_SIZE))
         igninfo_json = {'file': 'images/ignition.img'}
 
     # Generate JSON file that lists OS features available to
@@ -512,7 +574,7 @@ def generate_iso():
         assert initramfs_size % 4 == 0
 
         # combine kernel, initramfs and cmdline using the mk-s390image tool
-        os.truncate(iso_initramfs, initramfs_size + ignition_img_size)
+        os.truncate(iso_initramfs, initramfs_size + IGNITION_IMG_SIZE)
         runcmd(['/usr/bin/mk-s390image',
                kernel_dest,
                os.path.join(tmpisoimages, 'cdboot.img'),
@@ -538,7 +600,7 @@ def generate_iso():
             igninfo_json = {
                 'file': 'images/cdboot.img',
                 'offset': offset + initramfs_size,
-                'length': ignition_img_size,
+                'length': IGNITION_IMG_SIZE,
             }
 
             # kargs are part of 'images/cdboot.img' blob
@@ -707,7 +769,7 @@ boot
 
     miniso_data = os.path.join(tmpisocoreos, "miniso.dat")
     with open(miniso_data, 'wb') as f:
-        f.truncate(miniso_data_file_size)
+        f.truncate(MINISO_DATA_FILE_SIZE)
 
     if args.fixture:
         # Replace or delete anything irrelevant to coreos-installer
@@ -764,13 +826,6 @@ boot
         }
     })
     shutil.move(tmpisofile, f"{builddir}/{iso_name}")
-
-    kernel_name = f'{base_name}-{args.build}-live-kernel-{basearch}'
-    initramfs_name = f'{base_name}-{args.build}-live-initramfs.{basearch}.img'
-    rootfs_name = f'{base_name}-{args.build}-live-rootfs.{basearch}.img'
-    kernel_file = os.path.join(builddir, kernel_name)
-    initramfs_file = os.path.join(builddir, initramfs_name)
-    rootfs_file = os.path.join(builddir, rootfs_name)
     shutil.copyfile(os.path.join(tmpisoimagespxe, kernel_img), kernel_file)
     shutil.move(pxe_initramfs, initramfs_file)
     shutil.move(pxe_rootfs, rootfs_file)
@@ -801,7 +856,24 @@ with open(build_semaphore, 'w') as f:
     f.write(f"{time.time_ns()}")
 
 try:
-    generate_iso()
+    if COSA_OSBUILD_LIVEISO == "":
+        generate_iso()
+    else:
+        command = [
+            "/usr/bin/cosa", "supermin-run",
+            "--cache",  "/usr/lib/coreos-assembler/runvm-osbuild",
+            "--config",
+            image_for_disk_json,
+            "--mpp",
+            f"/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.{basearch}.mpp.yaml",
+            "--filepath",
+            tmpisofile
+        ]
+        subprocess.run(command)
+        update_buildmeta()
+        # Extract live artifacts from ISO. OSBUILD does not support an output with multiple files
+        command = ["coreos-installer", "iso", "extract", "pxe", tmpisofile]
+        subprocess.run(command)
 finally:
     if os.path.exists(build_semaphore):
         os.unlink(build_semaphore)

src/cmdlib.sh

@@ -730,6 +730,9 @@ runvm() {
 
     # include COSA in the image
     find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.dmverity >> "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso >> "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso.meta.json >> "${vmpreparedir}/hostfiles"
 
     # and include all GPG keys
     find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles"

src/deps-x86_64.txt

@@ -1,6 +1,3 @@
-# For generating ISO images
-syslinux-nonlinux
-
 # For the pipeline and developers to interact with AWS. It's not needed by cosa
 # itself. This isn't available on s390x at least, so make it x86_64-only.
 awscli2

src/deps.txt

@@ -16,9 +16,6 @@ dumb-init
 rpm-ostree createrepo_c openssh-clients python3-createrepo_c composefs
 dnf-utils
 
-# For generating ISO images
-genisoimage
-
 # Standard build tools
 make git rpm-build
 
@@ -100,4 +97,4 @@ bsdtar
 fedora-repos-ostree
 
 # For graphing manifest includes using `manifest_graph`
-python-anytree
+python-anytree

src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml

@@ -11,13 +11,17 @@ mpp-vars:
   extra_kargs: $extra_kargs
   metal_image_size_mb: $metal_image_size_mb
   cloud_image_size_mb: $cloud_image_size_mb
+  squashfs_compression: $squashfs_compression
   bios_boot_size_mb: 1
   ppc_prep_size_mb: 4
   reserved_part_size_mb: 1
   efi_system_size_mb: 127
   boot_size_mb: 384
   sector_size: 512
   four_k_sector_size: 4096
+  metal_filename: $metal_filename
+  metal4k_filename: $metal4k_filename
+  live_efiboot_img_size_mb: $live_efiboot_img_size_mb
   # Filesystem UUID and label definitions. These UUIDs
   # are looked for on boot and if found replaced with
   # a new random UUID to make each install unique.
@@ -593,3 +597,5 @@ pipelines:
       path: platform.metal.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.live-iso.ipp.yaml

src/osbuild-manifests/platform.live-iso.ipp.yaml

@@ -0,0 +1,96 @@
+# This file defines the pipeline for building the live ISO.
+version: '2'
+pipelines:
+  - name: live-iso
+    stages:
+      # Yuck: here we copy the metal and metal4k images into the tree from the
+      # outside instead of referencing the metal/metal4k osbuild pipelines.
+      # The reason for that is that our tooling and pipeline is currently
+      # very oriented towards building the metal images as separate artifacts
+      # in separate invocations of osbuild, so we need to support the metal
+      # images pre-existing. Also, the invocations in which we currently build
+      # those images are done with -snapshot on the cache qcow so it can be
+      # parallelized, so we can't just checkpoint those stage outputs. And even
+      # if we did, it doesn't feel quite right to rely on whatever is cached
+      # vs the canonical images. I think to improve this, we should move to
+      # building the metal, metal4k and live artifacts in one invocation.
+      #
+      # The mpp-if here is so osbuild-mpp doesn't try to embed the (possibly
+      # non-existent metal images) when building other stuff... We really should
+      # improve how we interact with osbuild.
+      - mpp-if: metal_filename != ''
+        then:
+          type: org.osbuild.copy
+          inputs:
+            metal:
+              type: org.osbuild.files
+              origin: org.osbuild.source
+              mpp-embed:
+                id: metal
+                url:
+                  mpp-format-string: 'file://{metal_filename}'
+          options:
+            paths:
+              - from:
+                  mpp-format-string: input://metal/{embedded['metal']}
+                to: tree:///metal.raw
+      - mpp-if: metal4k_filename != ''
+        then:
+          type: org.osbuild.copy
+          inputs:
+            metal4k:
+              type: org.osbuild.files
+              origin: org.osbuild.source
+              mpp-embed:
+                id: metal4k
+                url:
+                  mpp-format-string: 'file://{metal4k_filename}'
+          options:
+            paths:
+              - from:
+                  mpp-format-string: input://metal4k/{embedded['metal4k']}
+                to: tree:///metal4k.raw
+      # We need to be able to create efiboot.img, a FAT filesystem image. It's
+      # hard to setup loopback devices from within the stage, so just do it
+      # here. This should normally be conditional on the architecture but it
+      # doesn't hurt either since it's a tiny file.
+      - type: org.osbuild.truncate
+        options:
+          filename: efiboot.img
+          size:
+            mpp-format-string: '{live_efiboot_img_size_mb * 1024 * 1024}'
+      - type: org.osbuild.coreos.live-iso
+        inputs:
+          tree:
+            type: org.osbuild.tree
+            origin: org.osbuild.pipeline
+            references:
+              # XXX: we should rename this pipeline now that we don't use it as
+              # a buildroot
+              - name:build
+        devices:
+          metal:
+            type: org.osbuild.loopback
+            options:
+              filename: metal.raw
+              partscan: true
+              read-only: true
+              sector-size: 512
+          metal4k:
+            type: org.osbuild.loopback
+            options:
+              filename: metal4k.raw
+              partscan: true
+              read-only: true
+              sector-size: 4096
+          efiboot_img:
+            type: org.osbuild.loopback
+            options:
+              filename: efiboot.img
+        options:
+          efiboot_img_filename:
+            mpp-format-string: 'efiboot.img'
+          filename:
+            mpp-format-string: '{filename}'
+          squashfs_compression:
+            mpp-format-string: '{squashfs_compression}'