Merge branch 'master' into run-oci-bundles_1754

NOTICE

@@ -0,0 +1,20 @@
+Charliecloud is copyright © Triad National Security, LLC and others.
+
+This software was produced in part under U.S. Government contract
+89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated
+by Triad National Security, LLC for the U.S. Department of Energy/National
+Nuclear Security Administration.
+
+The Government is granted for itself and others acting on its behalf a
+nonexclusive, paid-up, irrevocable worldwide license in this material to
+reproduce, prepare derivative works, distribute copies to the public, perform
+publicly and display publicly, and to permit others to do so.
+
+Neither the government nor Triad National Security, LLC makes any warranty,
+express or implied, or assumes any liability for use of this software.
+
+If software is modified to produce derivative works, such derivative works
+should be clearly marked, so as not to confuse it with the version available
+from LANL.
+
+LA-CC 14-096

README.rst

@@ -136,32 +136,15 @@ Other publications:
     Supercomputing*. DOI: `10.1145/3624062.3624585
     <https://doi.org/10.1145/3624062.3624585>`_.
 
-Copyright and license
----------------------
-
-Charliecloud is copyright © 2014–2023 Triad National Security, LLC and others.
-
-This software was produced under U.S. Government contract 89233218CNA000001
-for Los Alamos National Laboratory (LANL), which is operated by Triad National
-Security, LLC for the U.S. Department of Energy/National Nuclear Security
-Administration.
-
-This is open source software (LA-CC 14-096); you can redistribute it and/or
-modify it under the terms of the Apache License, Version 2.0. A copy is
-included in file LICENSE. You may not use this software except in compliance
-with the license.
-
-The Government is granted for itself and others acting on its behalf a
-nonexclusive, paid-up, irrevocable worldwide license in this material to
-reproduce, prepare derivative works, distribute copies to the public, perform
-publicly and display publicly, and to permit others to do so.
+License and intellectual property
+---------------------------------
 
-Neither the government nor Triad National Security, LLC makes any warranty,
-express or implied, or assumes any liability for use of this software.
+Charliecloud is open source software; you can redistribute it and/or modify it
+under the terms of the Apache License, Version 2.0. A copy is included in file
+LICENSE. You may not use this software except in compliance with the license.
 
-If software is modified to produce derivative works, such derivative works
-should be clearly marked, so as not to confuse it with the version available
-from LANL.
+Copyrights and patents are retained by contributors. No copyright assignment
+is required to contribute to Charliecloud.
 
 
 ..  LocalWords:  USENIX's CNA Meisam figshare

VERSION

@@ -1 +1 @@
-0.38~pre
+0.39~pre

bin/ch-checkns.c

@@ -71,7 +71,7 @@ void fatal_(const char *file, int line, int errno_, const char *str)
    char *url = "https://github.com/hpc/charliecloud/blob/master/bin/ch-checkns.c";
    printf("error: %s: %d: %s\n", file, line, str);
    printf("errno: %d\nsee: %s\n", errno_, url);
-   exit(1);
+   exit(EXIT_MISC_ERR);
 }
 
 int main(int argc, char *argv[])

bin/ch-completion.bash

@@ -259,14 +259,16 @@ _ch_convert_complete () {
 _image_build_opts="-b --bind --build-arg -f --file --force
                    --force-cmd -n --dry-run --parse-only -t --tag"
 
+_image_modify_opts="-c -S --shell"
+
 _image_common_opts="-a --arch --always-download --auth --break
                     --cache --cache-large --dependencies -h --help
                     --no-cache --no-lock --no-xattrs --profile
                     --rebuild --password-many -q --quiet -s --storage
                     --tls-no-verify -v --verbose --version --xattrs"
 
-_image_subcommands="build build-cache delete gestalt
-                    import list pull push reset undelete"
+_image_subcommands="build build-cache delete gestalt import
+                    list modify pull push reset undelete"
 
 # archs taken from ARCH_MAP in charliecloud.py
 _archs="amd64 arm/v5 arm/v6 arm/v7 arm64/v8 386 mips64le ppc64le s390x"
@@ -374,20 +376,26 @@ _ch_image_complete () {
     build-cache)
         COMPREPLY=( $(compgen -W "--reset --gc --tree --dot" -- "$cur") )
         ;;
-    delete|list)
-        if [[ "$sub_cmd" == "list" ]]; then
+    delete|list|modify)
+        case "$sub_cmd" in
+        list)
             if [[ "$prev" == "--undeletable" || "$prev" == "--undeleteable" || "$prev" == "-u" ]]; then
                 COMPREPLY=( $(compgen -W "$(_ch_undelete_list "$strg_dir")" -- "$cur") )
                 return 0
             fi
-            extras+="$extras -l --long -u --undeletable"
+            extras="$extras -l --long -u --undeletable"
             # If “cur” starts with “--undelete,” add “--undeleteable” (the less
             # correct version of “--undeletable”) to the list of possible
             # completions.
             if [[ ${cur::10} == "--undelete" ]]; then
                 extras="$extras --undeleteable"
             fi
-        fi
+            ;;
+        modify)
+            # FIXME: Implement
+            extras="$extras $_image_modify_opts"
+            ;;
+        esac
         COMPREPLY=( $(compgen -W "$(_ch_list_images "$strg_dir") $extras" -- "$cur") )
         __ltrim_colon_completions "$cur"
         ;;

bin/ch-image.py.in

@@ -13,7 +13,9 @@ import charliecloud as ch
 import build
 import build_cache as bu
 import filesystem as fs
+import image as im
 import misc
+import modify
 import pull
 import push
 
@@ -278,6 +280,34 @@ def main():
    sp.add_argument("image_ref", metavar="IMAGE_REF", nargs="?",
                    help="print details of this image only")
 
+   # modify
+   sp = ap.add_parser("modify", "foo")
+   add_opts(sp, modify.main, deps_check=True, stog_init=True)
+   sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1,
+                   help="Run CMD as though specified by a RUN instruction. Can be repeated.")
+   sp.add_argument("-i", "--interactive", action="store_true",
+                   help="modify in interactive mode, even if stdin is not a TTY")
+   sp.add_argument("-S", "--shell", metavar="shell", default="/bin/sh",
+                   help="use SHELL instead of the default /bin/sh")
+   sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify")
+   sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image")
+   sp.add_argument("script", metavar="SCRIPT", help="foo", nargs='?')
+   # Options “modify” shares with “build”. Note that while we could abstract
+   # this out to avoid repeated lines, as we do for “common_opts”, we’ve decided
+   # that the tradeoff in code readability wouldn’t be worth it.
+   sp.add_argument("-b", "--bind", metavar="SRC[:DST]",
+                   action="append", default=[],
+                   help="mount SRC at guest DST (default: same as SRC)")
+   sp.add_argument("--build-arg", metavar="ARG[=VAL]",
+                   action="append", default=[],
+                   help="set build-time variable ARG to VAL, or $ARG if no VAL")
+   sp.add_argument("--force", metavar="MODE", nargs="?", default="seccomp",
+                   type=ch.Force_Mode, const="seccomp",
+                   help="inject unprivileged build workarounds")
+   sp.add_argument("--force-cmd", metavar="CMD,ARG1[,ARG2...]",
+                   action="append", default=[],
+                   help="command arg(s) to add under --force=seccomp")
+
    # pull
    sp = ap.add_parser("pull",
                       "copy image from remote repository to local filesystem")

bin/ch-run.c

@@ -228,7 +228,7 @@ int main(int argc, char *argv[])
 
    if (arg_next >= argc - 1) {
       printf("usage: ch-run [OPTION...] IMAGE -- COMMAND [ARG...]\n");
-      FATAL("IMAGE and/or COMMAND not specified");
+      FATAL(0, "IMAGE and/or COMMAND not specified");
    }
    args.c.img_ref = argv[arg_next++];
    args.c.newroot = realpath_(args.c.newroot, true);
@@ -262,11 +262,11 @@ int main(int argc, char *argv[])
       break;
    case IMG_SQUASH:
 #ifndef HAVE_LIBSQUASHFUSE
-      FATAL("this ch-run does not support internal SquashFS mounts");
+      FATAL(0, "this ch-run does not support internal SquashFS mounts");
 #endif
       break;
    case IMG_NONE:
-      FATAL("unknown image type: %s", args.c.img_ref);
+      FATAL(0, "unknown image type: %s", args.c.img_ref);
       break;
    }
 
@@ -497,7 +497,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state)
 #ifdef HAVE_FNM_EXTMATCH
          exit(0);
 #else
-         exit(1);
+         exit(EXIT_MISC_ERR);
 #endif
       } else if (!strcmp(arg, "overlayfs")) {
 #ifdef HAVE_OVERLAYFS
@@ -509,13 +509,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state)
 #ifdef HAVE_SECCOMP
          exit(0);
 #else
-         exit(1);
+         exit(EXIT_MISC_ERR);
 #endif
       } else if (!strcmp(arg, "squash")) {
 #ifdef HAVE_LIBSQUASHFUSE
          exit(0);
 #else
-         exit(1);
+         exit(EXIT_MISC_ERR);
 #endif
       } else if (!strcmp(arg, "tmpfs-xattrs")) {
 #ifdef HAVE_TMPFS_XATTRS
@@ -525,7 +525,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state)
 #endif
       }
       else
-         FATAL("unknown feature: %s", arg);
+         FATAL(0, "unknown feature: %s", arg);
       break;
    case -12: // --home
       Tf (args->c.host_home = getenv("HOME"), "--home failed: $HOME not set");
@@ -556,7 +556,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state)
       else if (!strcmp(arg, "log-fail"))
          test_logging(true);
       else
-         FATAL("invalid --test argument: %s; see source code", arg);
+         FATAL(0, "invalid --test argument: %s; see source code", arg);
       break;
    case 'b': {  // --bind
          char *src, *dst;

bin/ch-test

@@ -299,16 +299,17 @@ pedantry_set () {
     if [[ $ch_pedantic == no ]]; then
         ch_pedantic=  # proper boolean
     fi
-    # The motivation here is that in pedantic mode, we want to run all the
-    # tests we reasonably can. So, if the user *has* sudo, then default --sudo
-    # to yes. What is a little awkward is that “sudo -v” can generate a
-    # password prompt in the middle of the status output. An alternative is
-    # “sudo -nv”, which doesn’t; drawbacks are that you have to analyze the
-    # output (not exit code) and it generates a failed password log message if
-    # there is not already a sudo session going.
+    # Motivation here: In pedantic mode, we want to run all the tests we
+    # reasonably can. So, if the user *has* sudo, then default --sudo to yes.
+    # What is a little awkward is that “sudo true” can generate a password
+    # prompt in the middle of the status output. An alternative is “sudo -nv”,
+    # which doesn’t; drawbacks are that you have to analyze the output (not
+    # exit code) and it generates a failed password log message if there is
+    # not already a sudo session going. We also used to use “sudo -v”, which
+    # prompts for a password even if you have passwordless sudo set up.
     if    [[ -n $ch_pedantic ]] \
        && command -v sudo > /dev/null \
-       && sudo -v > /dev/null 2>&1; then
+       && sudo true > /dev/null 2>&1; then
         use_sudo_default=yes
     else
         use_sudo_default=
@@ -986,17 +987,6 @@ if [[ -z $phase ]]; then
 fi
 printf '\n'
 
-# See issue #1580
-# shellcheck disable=SC2016
-if [[ -d /var/tmp/img ]] || [[ -d /var/tmp/tar ]]; then
-printf '\n'
-  warning 'NOTE: default image and pack directories changed to:'
-  warning '      CH_TEST_IMGDIR=/var/tmp/${USER}.img'
-  warning '      CH_TEST_TARDIR=/var/tmp/${USER}.pack'
-fi
-
-printf '\n'
-
 #    variable name    CLI              environment         default
 #                     desc. width  description
 vset CH_TEST_SCOPE    "$scope"         "$CH_TEST_SCOPE"    standard \

bin/ch_core.c

@@ -235,7 +235,7 @@ void bind_mount(const char *src, const char *dst, enum bind_dep dep,
    if (!path_exists(dst_full, NULL, true))
       switch (dep) {
       case BD_REQUIRED:
-         FATAL("can't bind: destination not found: %s", dst_full);
+         FATAL(0, "can't bind: destination not found: %s", dst_full);
          break;
       case BD_OPTIONAL:
          return;
@@ -422,7 +422,7 @@ enum img_type image_type(const char *ref, const char *storage_dir)
       return IMG_SQUASH;
 
    // Well now we’re stumped.
-   FATAL("unknown image type: %s", ref);
+   FATAL(0, "unknown image type: %s", ref);
 }
 
 char *img_name2path(const char *name, const char *storage_dir)
@@ -568,7 +568,8 @@ void run_user_command(char *argv[], const char *initial_dir)
    if (verbose < LL_STDERR)
       T_ (freopen("/dev/null", "w", stderr));
    execvp(argv[0], argv);  // only returns if error
-   Tf (0, "can't execve(2): %s", argv[0]);
+   ERROR(errno, "can't execve(2): %s", argv[0]);
+   exit(EXIT_CMD);
 }
 
 /* Set up the fake-syscall seccomp(2) filter. This computes and installs a

bin/ch_fuse.c

@@ -188,7 +188,7 @@ int sq_loop(void)
 
    // Clean up zombie child if exit signal was SIGCHLD.
    if (!sigchld_received)
-      exit_code = 0;
+      exit_code = EXIT_SQUASH;
    else {
       Tf (wait(&child_status) >= 0, "can't wait for child");
       if (WIFEXITED(child_status)) {
@@ -203,7 +203,7 @@ int sq_loop(void)
          //
          // [1]: https://codereview.stackexchange.com/a/109349
          // [2]: https://man7.org/linux/man-pages/man2/wait.2.html
-         exit_code = 1;
+         exit_code = 128 + WTERMSIG(child_status);
          VERBOSE("child terminated by signal %d", WTERMSIG(child_status))
       }
    }
@@ -254,7 +254,7 @@ void sq_mount(const char *img_path, char *mountpt)
                                    &OPS, sizeof(OPS), sq.ll)) {
          break;  // success
       } else if (i <= 0) {
-         FATAL("too many FUSE errors; giving up");
+         FATAL(0, "too many FUSE errors; giving up");
       } else {
          WARNING("FUSE error mounting SquashFS; will retry");
          sleep(1);

bin/ch_misc.c

@@ -449,7 +449,7 @@ void test_logging(bool fail) {
    INFO("info");
    WARNING("warning");
    if (fail)
-      FATAL("the program failed inexplicably (\"log-fail\" specified)");
+      FATAL(0, "the program failed inexplicably (\"log-fail\" specified)");
    exit(0);
 }
 
@@ -586,6 +586,22 @@ void msg(enum log_level level, const char *file, int line, int errno_,
    va_end(ap);
 }
 
+void msg_error(const char *file, int line, int errno_,
+               const char *fmt, ...)
+{
+   va_list ap;
+
+   va_start(ap, fmt);
+   /* We print errors at LL_FATAL because, according to our documentation,
+      errors are never suppressed. Perhaps we need to rename this log level (see
+      issue #1914). */
+   msgv(LL_FATAL, file, line, errno_, fmt, ap);
+   va_end(ap);
+}
+
+/* Note that msg_fatal doesn’t call msg_error like we do in the Python code
+   because the variable number of arguments make it easier to simply define
+   separate functions. */
 noreturn void msg_fatal(const char *file, int line, int errno_,
                        const char *fmt, ...)
 {
@@ -595,7 +611,7 @@ noreturn void msg_fatal(const char *file, int line, int errno_,
    msgv(LL_FATAL, file, line, errno_, fmt, ap);
    va_end(ap);
 
-   exit(EXIT_FAILURE);
+   exit(EXIT_MISC_ERR);
 }
 
 /* va_list form of msg(). */