From e97bcc16c8be6dfe2fba225cde0cbc7281ca2e3d Mon Sep 17 00:00:00 2001 From: Gunter Miegel Date: Mon, 24 Jan 2022 22:40:19 +0100 Subject: [PATCH] 34 zstd image compression (#51) * ZSTD compression for inner initrd * Let zstd autodetect the number of threads to use * HWE kernel 5.11 w/ support of zstd compressed initrd * Upstream Loki config * Update existing symlinks for initrd and kernel * Add removing of gcc-9 and dependent packages * Remove all locale definitions beside en_US * Reduce overhead by locales * Remove libicu66 * Set KERNEL fallback variable to 5.11.0-46-generic --- .github/workflows/build_and_run.yaml | 4 +- debirf/create_rootfs_and_kernel | 6 +- debirf/debirf | 114 +++++++++++------- debirf/profiles/coinboot/debirf.conf | 7 +- .../coinboot/modules/a1_install-kernel | 16 ++- .../coinboot/modules/a2_r8168-network-driver | 15 ++- .../coinboot/modules/z0_remove-locales | 41 +++++-- .../profiles/coinboot/modules/z1_clean-root | 6 +- server/conf/loki/loki.yaml | 58 ++------- server/docker/coinboot-download-helper | 2 +- server/run_coinboot | 1 + 11 files changed, 148 insertions(+), 122 deletions(-) diff --git a/.github/workflows/build_and_run.yaml b/.github/workflows/build_and_run.yaml index c9196cb2..e8bacb9d 100644 --- a/.github/workflows/build_and_run.yaml +++ b/.github/workflows/build_and_run.yaml @@ -7,7 +7,7 @@ jobs: strategy: matrix: include: - - kernel: "5.4.0-58-generic" + - kernel: "5.11.0-46-generic" suite: "focal" steps: - name: Checkout repository @@ -75,7 +75,7 @@ jobs: run: ./setup_coinboot_requirements - name: Run Coinboot server and boot workers env: - KERNEL: "5.4.0-58-generic" + KERNEL: "5.11.0-46-generic" SUITE: "focal" shell: bash run: | diff --git a/debirf/create_rootfs_and_kernel b/debirf/create_rootfs_and_kernel index 15fe2e1d..07f5a68e 100755 --- a/debirf/create_rootfs_and_kernel +++ b/debirf/create_rootfs_and_kernel @@ -22,7 +22,7 @@ # Debian package to be used. apt update -apt install --yes debirf bc sudo +apt install --yes debirf bc sudo zstd # We have a customized version of debirf # with an adapted 'init' script able to @@ -63,5 +63,5 @@ time debirf make -n --root-build --no-warning /tmp/coinboot sudo install -Dv /tmp/coinboot/vmlinuz-* /mnt/build/${DEBIRF_SUITE}/coinboot-vmlinuz-${DEBIRF_KERNEL} sudo install -Dv /tmp/coinboot/*.cgz /mnt/build/${DEBIRF_SUITE}/coinboot-initramfs-${DEBIRF_KERNEL} -sudo ln -srv /mnt/build/${DEBIRF_SUITE}/coinboot-vmlinuz-${DEBIRF_KERNEL} /mnt/build/coinboot-vmlinuz-${DEBIRF_KERNEL} -sudo ln -srv /mnt/build/${DEBIRF_SUITE}/coinboot-initramfs-${DEBIRF_KERNEL} /mnt/build/coinboot-initramfs-${DEBIRF_KERNEL} +sudo ln -sfrv /mnt/build/${DEBIRF_SUITE}/coinboot-vmlinuz-${DEBIRF_KERNEL} /mnt/build/coinboot-vmlinuz-${DEBIRF_KERNEL} +sudo ln -sfrv /mnt/build/${DEBIRF_SUITE}/coinboot-initramfs-${DEBIRF_KERNEL} /mnt/build/coinboot-initramfs-${DEBIRF_KERNEL} diff --git a/debirf/debirf b/debirf/debirf index 9d88b40c..09cb7d72 100755 --- a/debirf/debirf +++ b/debirf/debirf @@ -101,7 +101,7 @@ EOF } create_debootstrap() { - msg "distro/suite: ${DEBIRF_DISTRO}/${DEBIRF_SUITE}" + msg "Distro/suite: ${DEBIRF_DISTRO}/${DEBIRF_SUITE}" local OPTS="--merged-usr --include=$INCLUDE${DEBIRF_EXTRA_PACKAGES:+,}$DEBIRF_EXTRA_PACKAGES --exclude=$EXCLUDE $DEBIRF_SUITE $DEBIRF_ROOT $DEBIRF_MIRROR" @@ -130,7 +130,7 @@ create_debootstrap() { # used with debootstrap (default non-privileged behavior) fix_dev() { if [ -L "$DEBIRF_ROOT"/dev -o ! -d "$DEBIRF_ROOT"/dev ] ; then - msg "fixing debirf root dev tree..." + msg "Fixing debirf root dev tree..." # remove old dev fakeroot_if_needed rm -f "$DEBIRF_ROOT"/dev @@ -157,7 +157,8 @@ pack_rootfs() { # abort with a failure if our attempts to build the rootfs fail: msg "pack_rootfs" set -o pipefail - fakeroot_if_needed bash -c ". $DEBIRF_COMMON && FAKECHROOT_EXCLUDE_PATH=/does-not-exist debirf_exec sh -c 'find * | grep -a -v -e ^run/ | cpio --create -H newc'" | gzip -9 > "$1" + #fakeroot_if_needed bash -c ". $DEBIRF_COMMON && FAKECHROOT_EXCLUDE_PATH=/does-not-exist debirf_exec sh -c 'find * | grep -a -v -e ^run/ | cpio --create -H newc'" | gzip -9 > "$1" + fakeroot_if_needed bash -c ". $DEBIRF_COMMON && FAKECHROOT_EXCLUDE_PATH=/does-not-exist debirf_exec sh -c 'find * | grep -a -v -e ^run/ | cpio --create -H newc'" | zstd -19 -T0 > "$1" } export -f pack_rootfs @@ -187,16 +188,33 @@ create_initrd_plugin() { # copy in busybox-static from root, and link executables #cp -f "$DEBIRF_ROOT"/bin/busybox "$NEST_ROOT"/bin/ for util in awk cpio cat free grep gunzip ls mkdir mount rm sh sleep umount modprobe mkfs.ext2 depmod; do - ln "$NEST_ROOT"/bin/busybox "$NEST_ROOT"/bin/"$util" + ln "$NEST_ROOT"/bin/busybox "$NEST_ROOT"/bin/"$util" done # copy in run-init and needed libraries cp -f /usr/lib/klibc/bin/run-init "$NEST_ROOT"/bin/ cp -f /lib/klibc-* "$NEST_ROOT"/lib/ + # Upstream ZSTD + wget https://github.com/facebook/zstd/releases/download/v1.5.1/zstd-1.5.1.tar.gz -P /tmp + echo "e28b2f2ed5710ea0d3a1ecac3f6a947a016b972b9dd30242369010e5f53d7002 /tmp/zstd-1.5.1.tar.gz" | sha256sum --check + tar -xvzf /tmp/zstd-1.5.1.tar.gz -C $DEBIRF_ROOT/tmp + debirf_exec apt update + debirf_exec apt install --yes build-essential + debirf_exec make -C /tmp/zstd-1.5.1/programs/ zstd-decompress + install -vD "$DEBIRF_ROOT/tmp/zstd-1.5.1/programs/zstd-decompress" "$NEST_ROOT"/"bin" + rm -rf $DEBIRF_ROOT/tmp/zstd* + debirf_exec apt purge --yes ^gcc-9 build-essential + debirf_exec apt autoremove --yes + debirf_exec apt autoclean + + + for libary in $(ldd "$NEST_ROOT"/bin/zstd-decompress | grep -oP '/lib.*\s+'); do + install -vD "$DEBIRF_ROOT"/"$libary" "$NEST_ROOT"/"$libary" + done + # Copy in the modules for ZRAM and for Zstandard # We need an recent package list to determine the following variables - debirf_exec apt-get update local KERNEL_VERS=$(ls -1 "${DEBIRF_ROOT}/lib/modules" | head -n1) local EXTRA_PACKAGE="linux-modules-extra-$KERNEL_VERS" local EXTRA_PACKAGE_ARCHIVE=$(debirf_exec apt-cache show $EXTRA_PACKAGE | grep -oP $EXTRA_PACKAGE.*deb) @@ -214,8 +232,11 @@ create_initrd_plugin() { local ZSTD_MODULE=/lib/modules/${KERNEL_VERS}/kernel/crypto/zstd.ko install -vD "$DEBIRF_ROOT/$ZSTD_MODULE" "$NEST_ROOT"/"$ZSTD_MODULE" + # Kernels above 5.4 have zstd compression in the kernel + if [[ $DEBIRF_KERNEL =~ '5.4' ]]; then local ZSTD_COMPRESS_MODULE=/lib/modules/${KERNEL_VERS}/kernel/lib/zstd/zstd_compress.ko install -vD "$DEBIRF_ROOT/$ZSTD_COMPRESS_MODULE" "$NEST_ROOT"/"$ZSTD_COMPRESS_MODULE" + fi # symlink /lib64 if that is done on host # FIXME: is this necessary? @@ -252,12 +273,11 @@ fi MEMSIZE=\$(free | grep 'Mem:' | awk '{ print \$2 }') mkdir /newroot -# TODO: Try zstd as compression algorithm # TODO: Find out of using "mem_limit' makes sense # TODO: Fine tune mount options (atime ... ) # https://www.kernel.org/doc/Documentation/blockdev/zram.txt if (grep -q zram /proc/cmdline); then - echo "honoring zram kernel arg" + echo "Honoring zram kernel arg" depmod modprobe -v zram modprobe -v zstd @@ -279,8 +299,8 @@ if (grep -q break=preunpack /proc/cmdline); then fi cd /newroot -echo unpacking rootfs... -gunzip - < /rootfs.cgz | cpio -i +echo Unpacking rootfs... +zstd-decompress -d /rootfs.czst -c | cpio -i if (grep -q break=bottom /proc/cmdline); then echo "Honoring break=bottom kernel arg" @@ -299,7 +319,6 @@ EOF export PATH=/sbin:/usr/sbin:/bin:/usr/bin - mkdir -p /dev /proc /sys /run # Mount the /proc and /sys filesystems. mount -t proc none /proc @@ -327,7 +346,6 @@ export LC_ALL=en_US.UTF-8 export LANG=en_US.UTF-8 export LANGUAGE=en_US.UTF-8 - # Determine IP-address of HTTP-server that # delivers the plugin archives. # The server is the same as the one which provides the @@ -372,7 +390,6 @@ else # necessary links and cache to use them. # TODO: Found a way to do this only when necessary - e.g. when # libary is added by a plugin. - /sbin/ldconfig fi @@ -380,14 +397,18 @@ fi exec /sbin/init EOF chmod a+x "$DEBIRF_ROOT"/init2 - #pack_rootfs "$1" - msg "creating rootfs.cgz..." + msg "Creating rootfs.czst..." fakeroot_if_needed ln -sf /sbin/init "$DEBIRF_ROOT/init" - pack_rootfs "$NEST_ROOT"/rootfs.cgz + pack_rootfs "$NEST_ROOT"/rootfs.czst - msg "creating wrapper cgz..." - fakeroot_if_needed sh -c "cd $NEST_ROOT && find * | cpio --create -H newc" | gzip -9 > "$INITRD" + msg "Creating wrapper initrafms..." + # For 20.04 only .3 HWE Kernels support initrd with zstd compression + if [[ $DEBIRF_KERNEL =~ '5.4' ]]; then + fakeroot_if_needed sh -c "cd $NEST_ROOT && find * | cpio --create -H newc" | gzip -9 > "$INITRD" + else + fakeroot_if_needed sh -c "cd $NEST_ROOT && find * | cpio --create -H newc" | zstd -19 -T0 > "$INITRD" + fi } # initrd method: nested cpio archives @@ -420,6 +441,7 @@ create_initrd_nested() { # create nest init cat > "$NEST_ROOT"/init < ./dev/console EOF + chmod a+x "$NEST_ROOT"/init - msg "creating rootfs.cgz..." + msg "Creating rootfs.cgz..." fakeroot_if_needed ln -sf /sbin/init "$DEBIRF_ROOT/init" pack_rootfs "$NEST_ROOT"/rootfs.cgz @@ -454,8 +477,7 @@ EOF fakeroot_if_needed sh -c "cd $NEST_ROOT && find * | cpio --create -H newc" | gzip -9 > "$INITRD" } -# determine what the host system distro is, and set debirf build defaults -# accordingly +# Determine what the host system distro is, set debirf build defaults accordingly set_distro() { local distro local suite @@ -492,7 +514,7 @@ setup_environment() { # check profile if [ -d "$DEBIRF_PROFILE" ] ; then - msg "loading profile '$DEBIRF_PROFILE_NAME'..." + msg "Loading profile '$DEBIRF_PROFILE_NAME'..." DEBIRF_CONF="$DEBIRF_PROFILE/debirf.conf" DEBIRF_MODULES="$DEBIRF_PROFILE/modules" else @@ -551,7 +573,7 @@ setup_environment() { # check variables if [ "$CHECK_VARS" ] ; then - msg "debirf variables:" + msg "Debirf variables:" env | /bin/grep "^DEBIRF_" read -p "enter to continue: " OK fi @@ -642,7 +664,7 @@ EOF if [ "$DEBIRF_KERNEL_PACKAGE" ] ; then if [ -f "$DEBIRF_KERNEL_PACKAGE" ] ; then - msg "using kernel package '$DEBIRF_KERNEL_PACKAGE'" + msg "Using kernel package '$DEBIRF_KERNEL_PACKAGE'" else failure "Kernel package '$DEBIRF_KERNEL_PACKAGE' not found." fi @@ -653,7 +675,7 @@ EOF # determine write mode if [ -d "$DEBIRF_ROOT" ] ; then if [ -z "$WRITE_MODE" ] ; then - msg "debirf root already exists. select one of the following:" + msg "Debirf root already exists. select one of the following:" CASE1='new: delete the old root and create a new one' CASE2='overwrite: leave the old root and debootstrap on top of it' CASE3='skip: skip building the root and go right to installing modules' @@ -681,23 +703,23 @@ EOF fi case "$WRITE_MODE" in 'new') - msg "creating debirf root..." + msg "Creating debirf root..." > "$DEBIRF_FAKEROOT_STATE" create_debootstrap ;; 'rewrite') - msg "clearing old debirf root..." + msg "Clearing old debirf root..." rm -rf "$DEBIRF_ROOT" - msg "creating debirf root..." + msg "Creating debirf root..." > "$DEBIRF_FAKEROOT_STATE" create_debootstrap ;; 'overwrite') - msg "overwriting old debirf root..." + msg "Overwriting old debirf root..." create_debootstrap ;; 'skip') - msg "skipping debootstrap..." + msg "Skipping debootstrap..." ;; *) failure "aborting." @@ -708,7 +730,7 @@ EOF fix_dev else - msg "not building root" + msg "Not building root" fi ### END BUILD ROOT @@ -716,9 +738,9 @@ EOF if [ "$STAGE_MODULES" = 'true' ] ; then msg "executing modules..." run_modules - msg "modules complete" + msg "Modules complete" else - msg "not running modules" + msg "Not running modules" fi ### END RUN MODULES @@ -731,18 +753,18 @@ EOF local KERNEL_VERS=$(ls -1 "${DEBIRF_ROOT}/lib/modules" | head -n1) local INITRD="${DEBIRF_LABEL}_${DEBIRF_SUITE}_${KERNEL_VERS}.cgz" - msg "creating debirf initrd ('$DEBIRF_METHOD')..." + msg "Creating debirf initrd ('$DEBIRF_METHOD')..." create_initrd_${DEBIRF_METHOD} "${DEBIRF_BUILDD}/${INITRD}" # final output local KERNEL=$(ls "$DEBIRF_BUILDD" | grep "vmlinu" | grep "$KERNEL_VERS$") - msg "debirf initrd created." + msg "Debirf initrd created." if [ "${DEBIRF_BUILDD}/${KERNEL}" ] ; then - msg "kernel: ${DEBIRF_BUILDD}/${KERNEL}" + msg "Kernel: ${DEBIRF_BUILDD}/${KERNEL}" fi - msg "initrd: ${DEBIRF_BUILDD}/${INITRD}" + msg "Initrd: ${DEBIRF_BUILDD}/${INITRD}" else - msg "not creating initramfs." + msg "Not creating initramfs." fi ### END BUILD INITRD } @@ -774,7 +796,7 @@ makeiso() { failure "Failed to find a kernel. Maybe you need to run 'debirf make $DEBIRF_PROFILE' first?" ;; 1) - msg "kernel found." + msg "Kernel found." local KERNEL=${KERNEL:-$(cd "${DEBIRF_BUILDD}" && ls -1 vmlinu*)} ;; *) @@ -788,7 +810,7 @@ makeiso() { failure "Failed to find a single initramfs. Maybe you need to run 'debirf make $DEBIRF_PROFILE' first?" ;; 1) - msg "initramfs found." + msg "Initramfs found." local INITRAMFS=${INITRAMFS:-$(cd "${DEBIRF_BUILDD}" && ls -1 "${DEBIRF_LABEL}"_*.cgz)} ;; *) @@ -807,7 +829,7 @@ makeiso() { fi fi - msg "creating debirf iso..." + msg "Creating debirf iso..." # determine the iso name from the initramfs local ISO=$(basename "$INITRAMFS" .cgz).iso @@ -819,7 +841,7 @@ makeiso() { case "$DEBIRF_ISO_BOOTLOADER" in grub) - msg "using GRUB as bootloader..." + msg "Using GRUB as bootloader..." # use hard links to avoid massive copying time (we're almost certainly on the same filesystem) ln "${DEBIRF_BUILDD}/${KERNEL}" "${ISODIR}/" || failure "Failed to link kernel into iso" @@ -851,7 +873,7 @@ EOF ;; isolinux) - msg "using isolinux as bootloader..." + msg "Using isolinux as bootloader..." (which xorriso > /dev/null) || failure "xorriso is not in your path. Maybe you need to install it?" @@ -914,8 +936,8 @@ EOF # do we need to clean up the iso/ directory so that this can be run again? - msg "debirf iso created." - msg "iso: ${DEBIRF_BUILDD}/${ISO}" + msg "Debirf ISO created." + msg "ISO: ${DEBIRF_BUILDD}/${ISO}" } diff --git a/debirf/profiles/coinboot/debirf.conf b/debirf/profiles/coinboot/debirf.conf index 879afdb2..6cdf3f6c 100644 --- a/debirf/profiles/coinboot/debirf.conf +++ b/debirf/profiles/coinboot/debirf.conf @@ -58,11 +58,14 @@ DEBIRF_MIRROR=http://archive.ubuntu.com/ubuntu # #DEBIRF_EXTRA_PACKAGES= - # As we want to vary this variable for CI/CD purposes we check if # it is already set in the environment, if not set a default value. if [[ -z $DEBIRF_KERNEL ]]; then - DEBIRF_KERNEL=5.4.0-58-generic + #DEBIRF_KERNEL=5.4.0-58-generic + # HWE + DEBIRF_KERNEL=5.11.0-46-generic + # HWE edge + # DEBIRF_KERNEL=5.13.0-25-generic else echo "DEBIRF_KERNEL already set by environment to: $DEBIRF_KERNEL" fi diff --git a/debirf/profiles/coinboot/modules/a1_install-kernel b/debirf/profiles/coinboot/modules/a1_install-kernel index ab74d0ba..3e201aab 100755 --- a/debirf/profiles/coinboot/modules/a1_install-kernel +++ b/debirf/profiles/coinboot/modules/a1_install-kernel @@ -1,5 +1,4 @@ #!/bin/bash -e -set -x # debirf module: a1_install-kernel @@ -23,6 +22,14 @@ set -x # # Copyright (C) 2018, 2019, 2020 Gunter Miegel coinboot.io +# General note +# In contrast to a conventional disk-based Linux OS installation +# a Coinboot machine has from the perspective of package management +# no kernel or kernel modules installed. +# There is no kernel and initrd under /boot. +# /lib/modules/ is populated by unpacking the linux-modules- package +# by this the dependency to install the kernel package is avoided. + # clear out old modules if they exist, to avoid confusion rm -rf "$DEBIRF_ROOT/lib/modules" @@ -49,8 +56,6 @@ if [ -z "$DEBIRF_KERNEL_PACKAGE" ] ; then # to just use "apt-get download", but that's only supported since # wheezy. fi - #debirf_exec apt-get install -d -y "$KNAME" - # debirf_exec apt-get install -d -y --no-install-recommends "$KNAME" debirf_exec apt-get download "$KNAME" # Attention: For a Kernel > 4.4 Ubuntu kernel and modules are in separate packages. debirf_exec apt-get download linux-modules-$DEBIRF_KERNEL @@ -63,7 +68,7 @@ fi KPKG=$(basename "$DEBIRF_ROOT"/linux-image-*) # Before cosmic 'dpkg --extract /' was used instead of '--unpack'. -# But at cosmic it changed its behavior to cleanup destination directories +# But at cosmic it changed its behavior to cleanup destination directories # before extracting data to them which breaks the whole debootstraped rootfs. echo "Unpacking kernel package $KPKG..." debirf_exec dpkg --unpack "$KPKG" @@ -77,7 +82,8 @@ debirf_exec rm -v "$MODPKG" # install the module init tools, since they are needed for depmod # these are in kmod in current releases # also add 'cpio' which is need for packing the kernel -#debirf_exec apt-get install -y kmod cpio +# TODO: check if cpio and kmod utils like depmod and insmod can be provide +# by busybox. debirf_exec apt-get install -y linux-base cpio kmod # depmod to create module list diff --git a/debirf/profiles/coinboot/modules/a2_r8168-network-driver b/debirf/profiles/coinboot/modules/a2_r8168-network-driver index 2293212d..11892943 100755 --- a/debirf/profiles/coinboot/modules/a2_r8168-network-driver +++ b/debirf/profiles/coinboot/modules/a2_r8168-network-driver @@ -1,5 +1,4 @@ #!/bin/bash -e -set -x # debirf module: network-driver-r8168 @@ -31,9 +30,9 @@ set -x KERNEL_RELEASE=$DEBIRF_KERNEL R8168_PACKAGE=r8168-dkms -R8168_PACKAGE_VERSION=8.048.00-1_all +R8168_PACKAGE_VERSION=8.049.02-1_all R8168_PACKAGE_DEB="$R8168_PACKAGE"_"$R8168_PACKAGE_VERSION".deb -PACKAGES="linux-base linux-headers-$DEBIRF_KERNEL libelf-dev wget dkms" +PACKAGES="linux-base linux-headers-$DEBIRF_KERNEL libelf-dev wget dkms linux-modules-5.11.0-46-generic" # Update package list. debirf_exec sudo apt-get update @@ -48,8 +47,7 @@ sudo mount -t proc /proc "$DEBIRF_ROOT/"proc chroot "$DEBIRF_ROOT" sudo apt-get install --yes linux-base chroot "$DEBIRF_ROOT" sudo apt-get install --yes $PACKAGES -# debirf_exec sudo wget http://mirrors.kernel.org/ubuntu/pool/universe/r/r8168/r8168-dkms_8.046.00-1_all.deb -P /tmp -debirf_exec sudo wget http://mirrors.kernel.org/ubuntu/pool/universe/r/r8168/$R8168_PACKAGE_DEB -P /tmp +debirf_exec sudo wget https://mirrors.edge.kernel.org/ubuntu/pool/universe/r/r8168/$R8168_PACKAGE_DEB -P /tmp chroot "$DEBIRF_ROOT" dpkg -i /tmp/$R8168_PACKAGE_DEB debirf_exec sudo rm -v /tmp/$R8168_PACKAGE_DEB @@ -62,12 +60,17 @@ sudo umount "$DEBIRF_ROOT"/proc # Afterwards we have to rebuilt the dependency map with depmod. # TODO: Put compiled module at a better place than the root of the module directory. sudo find $DEBIRF_ROOT/var/lib/dkms -name r8168.ko -exec cp -v {} "$DEBIRF_ROOT"/lib/modules/"$KERNEL_RELEASE" \; -#sudo cp -v "$DEBIRF_ROOT"/var/lib/dkms/r8168/8.046.00/"$KERNEL_RELEASE"/x86_64/module/r8168.ko "$DEBIRF_ROOT"/lib/modules/"$KERNEL_RELEASE" # Since Eoan linux-check-removal is triggered when removing the package # with the currently running linux kernel and omits that operation. # We just replace linux-chec-remove with a simple 'exit 0'. echo 'exit 0' > "$DEBIRF_ROOT"/usr/bin/linux-check-removal +# As linux-modules- gets purged as dependency we have to make a backup +# of the modules directory that we restore after apt-get purge. +sudo mv "$DEBIRF_ROOT"/lib/modules "$DEBIRF_ROOT"/tmp/ + # Purge installed packages of this modules to free up the space. debirf_exec apt-get purge $PACKAGES $R8168_PACKAGE --yes + +sudo mv "$DEBIRF_ROOT"/tmp/modules "$DEBIRF_ROOT"/lib/ diff --git a/debirf/profiles/coinboot/modules/z0_remove-locales b/debirf/profiles/coinboot/modules/z0_remove-locales index 6e5bd00b..3a07d615 100755 --- a/debirf/profiles/coinboot/modules/z0_remove-locales +++ b/debirf/profiles/coinboot/modules/z0_remove-locales @@ -10,16 +10,39 @@ # They are Copyright 2007, and are all released under the GPL, # version 3 or later. # -# Copyright (C) 2018,2021 Gunter Miegel coinboot.io +# Copyright (C) 2018, 2021, 2022 Gunter Miegel coinboot.io cd "$DEBIRF_ROOT" -debirf_exec echo 'en_US.UTF-8 UTF-8' > etc/locale.gen -debirf_exec echo 'LANG="en_US.UTF-8"' > etc/default/locale -debirf_exec mkdir -vp var/lib/locales/supported.d -debirf_exec echo 'en_US.UTF-8 UTF-8' > var/lib/locales/supported.d/en -debirf_exec rm -rfv usr/lib/locale/* -debirf_exec locale-gen --purge -locale -a -v +echo 'en_US.UTF-8 UTF-8' > etc/locale.gen +echo 'LANG="en_US.UTF-8"' > etc/default/locale +mkdir -vp var/lib/locales/supported.d +echo 'en_US.UTF-8 UTF-8' > var/lib/locales/supported.d/en +rm -rfv usr/lib/locale/* +mv usr/share/i18n/locales tmp/ + +while read line; do + install -vD tmp/locales/$line usr/share/i18n/locales/$line +done << EOF +en_GB +en_US +i18n +i18n_ctype +iso14651_t1 +iso14651_t1_common +translit_circle +translit_cjk_compat +translit_combining +translit_compat +translit_font +translit_fraction +translit_narrow +translit_neutral +translit_small +translit_wide +EOF -find usr/share/locale -maxdepth 1 -mindepth 1 -type d ! -iname 'en_US' -execdir rm -rvf '{}' \+ +rm -rf tmp/locales +find usr/share/locale -maxdepth 1 -mindepth 1 -type d ! -name 'en_US' -execdir rm -rvf '{}' \+ +debirf_exec locale-gen --purge +debirf_exec locale -a -v diff --git a/debirf/profiles/coinboot/modules/z1_clean-root b/debirf/profiles/coinboot/modules/z1_clean-root index 305a2c63..0fc990f7 100755 --- a/debirf/profiles/coinboot/modules/z1_clean-root +++ b/debirf/profiles/coinboot/modules/z1_clean-root @@ -1,4 +1,6 @@ #!/bin/bash -e +set -x + # debirf module: clean-root # clean up debirf root # @@ -13,11 +15,11 @@ # They are Copyright 2007, and are all released under the GPL, # version 3 or later. # -# Copyright (C) 2018 Gunter Miegel coinboot.io +# Copyright (C) 2018,2022 Gunter Miegel coinboot.io # Additional packages which can be removed to save space. #debirf_exec apt-get purge --yes linux-headers-* libgcc1 gcc cpp-5 libgcc-5-dev cpp-6 cpp-8 libc6-dev gcc-8 libgcc-8-dev file libmagic1 -debirf_exec apt-get purge --yes linux-headers-* gcc cpp* libgcc-*-dev lib*dev file libmagic1 | grep -v 'is not installed, so not removed' +debirf_exec apt-get purge --yes ^gcc-9 cpp libmagic1 libicu66 | grep -v 'is not installed, so not removed' # Do the autoremove dance debirf_exec apt-get --yes --purge autoremove | grep -v 'is not installed, so not removed' diff --git a/server/conf/loki/loki.yaml b/server/conf/loki/loki.yaml index f4de3b55..d37ee60d 100644 --- a/server/conf/loki/loki.yaml +++ b/server/conf/loki/loki.yaml @@ -2,20 +2,19 @@ auth_enabled: false server: http_listen_port: 3100 + grpc_listen_port: 9096 -ingester: - lifecycler: - address: 127.0.0.1 - ring: - kvstore: - store: inmemory - replication_factor: 1 - final_sleep: 0s - chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed - max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h - chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first - chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) - max_transfer_retries: 0 # Chunk transfers disabled +common: + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory schema_config: configs: @@ -27,38 +26,5 @@ schema_config: prefix: index_ period: 24h -storage_config: - boltdb_shipper: - active_index_directory: /loki/boltdb-shipper-active - cache_location: /loki/boltdb-shipper-cache - cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space - shared_store: filesystem - filesystem: - directory: /loki/chunks - -compactor: - working_directory: /loki/boltdb-shipper-compactor - shared_store: filesystem - -limits_config: - reject_old_samples: true - reject_old_samples_max_age: 168h - -chunk_store_config: - max_look_back_period: 0s - -table_manager: - retention_deletes_enabled: false - retention_period: 0s - ruler: - storage: - type: local - local: - directory: /loki/rules - rule_path: /loki/rules-temp alertmanager_url: http://localhost:9093 - ring: - kvstore: - store: inmemory - enable_api: true diff --git a/server/docker/coinboot-download-helper b/server/docker/coinboot-download-helper index b1ab0103..695f848e 100755 --- a/server/docker/coinboot-download-helper +++ b/server/docker/coinboot-download-helper @@ -47,7 +47,7 @@ fi DOWNLOAD_URL=https://github.com/${GITHUB_REPO}/releases/download/${TAG} if [ -z $KERNEL ]; then - KERNEL=5.4.0-58-generic + KERNEL=5.11.0-46-generic fi VMLINUZ=coinboot-vmlinuz-$KERNEL INITRAMFS=coinboot-initramfs-$KERNEL diff --git a/server/run_coinboot b/server/run_coinboot index 0407995c..4e30e1f5 100755 --- a/server/run_coinboot +++ b/server/run_coinboot @@ -148,6 +148,7 @@ down_docker_compose() { wait_for_server_to_be_ready() { while ! [ $(curl --silent http://192.168.1.2 | jq length) -gt 4 ]; do curl --silent http://192.168.1.2 | jq '.[].name' + echo 'Waiting for Coinboot server to provide the necessary files' echo '------------' sleep 10 done