deal with Docker’s changed COPY behavior

doc/ch-image.rst

@@ -885,19 +885,19 @@ bug-compatible.
    at the 2nd level or deeper, the source directory’s metadata (e.g.,
    permissions) are copied to the destination directory. (Not documented.)
 
-5. If an object appears in both the source and destination, and is at the 2nd
-   level or deeper, and is of different types in the source and destination,
-   then the source object will overwrite the destination object. (Not
-   documented.) For example, if :code:`/tmp/foo/bar` is a regular file, and
-   :code:`/tmp` is the context directory, then the following Dockerfile
-   snippet will result in a *file* in the container at :code:`/foo/bar`
-   (copied from :code:`/tmp/foo/bar`); the directory and all its contents will
-   be lost.
-
-     .. code-block:: docker
-
-       RUN mkdir -p /foo/bar && touch /foo/bar/baz
-       COPY foo /foo
+5. If an object (a) appears in both the source and destination, (b) is at the
+   2nd level or deeper, and (c) is different file types in source and
+   destination, there are multiple behaviors depending on Docker version. We
+   have not fully characterized this, but we have observed the following with
+   both file to directory and directory to file. (Not documented.)
+
+   * *Docker without BuildKit as of late 2020* (also Podman 4.4.3): Source
+     object will overwrite the destination object.
+
+   * *Docker 24.0.5 with BuildKit*: Error.
+
+   That is, Docker’s behavior changed at some point. Charliecloud uses the
+   first behavior.
 
 We expect the following differences to be permanent:
 

examples/copy/Dockerfile

@@ -164,14 +164,25 @@ COPY dirA dir08a
 # Dest:    new directory, one level, trailing slash
 COPY dirA dir08b/
 
+# NOTE: Not currently tested (see #1707).
+
 # Source:  one directory
 # Dest:    existing file, 2nd level
 #
-# Note: While this fails if the existing file is at the top level (which we
-# verify in test/build/50_dockerfile.bats), if the existing file is at the 2nd
-# level, it's overwritten by the directory.
-RUN touch dir08a/dirCb
-COPY dirCa dir08a
+# WARNING: While directory source and existing file destination *at the top
+# level* fails consistently (we verify this in 50_dockerfiles.bats), there are
+# multiple behaviors if the destination isn’t at the top level (#1707).
+#
+#   1. (Charliecloud, Podman, Docker w/o BuildKit as of late 2020.) Existing
+#      file is overwritten by the directory.
+#
+#   2. (Docker w/ BuildKit as of mid-2023.) Error.
+#
+# That is, Docker’s behavior changed at some point. I suspect this is
+# undocumented but I haven’t looked very hard. For the time being, we just
+# don’t test it.
+#RUN touch dir08a/dirCb
+#COPY dirCa dir08a
 
 # Source:  two directories, explicit
 # Dest:    existing directory
@@ -269,37 +280,39 @@ RUN test $(stat -c '%A' dir19/dir19a2 | cut -c8-) != 'rwx' \
 
 ## Destination: Symlink, 2nd level.
 
-# Note: This behavior is DIFFERENT from the symlink at 1st level tests above
-# (recall we are trying to be bug-compatible with Docker).
-
-# Set up destination.
-RUN mkdir dir20 \
- && echo new > dir20/filex \
- && mkdir dir20/dirx \
- && for i in $(seq 4); do \
-           echo file$i > dir20/file$i \
-        && ln -s file$i dir20/s_file$i \
-        && mkdir dir20/dir$i \
-        && echo dir$i/file_ > dir20/dir$i/file_ \
-        && ln -s dir$i dir20/s_dir$i; \
-    done \
- && ls -lR dir20
-
-# Copy in the new directory tree. In all of these cases, the source simply
-# overwrites the destination; symlinks are not followed.
+# NOTE: Not currently tested (see #1707).
+#
+# # Note: This behavior is DIFFERENT from the symlink at 1st level tests above
+# # (recall we are trying to be bug-compatible with Docker).
 #
-#      name     source        destination
-#      -------  ------------  ------------
-#   1. s_file1  file          link to file
-#   2. s_dir1   file          link to dir
-#   3. s_file2  link to file  link to file
-#   4. s_dir2   link to file  link to dir
-#   5. s_file3  link to dir   link to file
-#   6. s_dir3   link to dir   link to dir
-#   7. s_file4  directory     link to file
-#   8. s_dir4   directory     link to dir
+# # Set up destination.
+# RUN mkdir dir20 \
+#  && echo new > dir20/filex \
+#  && mkdir dir20/dirx \
+#  && for i in $(seq 4); do \
+#            echo file$i > dir20/file$i \
+#         && ln -s file$i dir20/s_file$i \
+#         && mkdir dir20/dir$i \
+#         && echo dir$i/file_ > dir20/dir$i/file_ \
+#         && ln -s dir$i dir20/s_dir$i; \
+#     done \
+#  && ls -lR dir20
 #
-COPY dirG dir20/
+# # Copy in the new directory tree. In all of these cases, the source simply
+# # overwrites the destination; symlinks are not followed.
+# #
+# #      name     source        destination
+# #      -------  ------------  ------------
+# #   1. s_file1  file          link to file
+# #   2. s_dir1   file          link to dir
+# #   3. s_file2  link to file  link to file
+# #   4. s_dir2   link to file  link to dir
+# #   5. s_file3  link to dir   link to file
+# #   6. s_dir3   link to dir   link to dir
+# #   7. s_file4  directory     link to file
+# #   8. s_dir4   directory     link to dir
+# #
+# COPY dirG dir20/
 
 
 ## Wrap up; this output helps to build the expectations in test.bats.

examples/copy/test.bats

@@ -40,7 +40,6 @@ dir16/
 dir17/
 dir18/
 dir19/
-dir20/
 file1a
 file1b
 file2
@@ -131,13 +130,7 @@ fileAa
 fileAa
 
 ./dir08a:
-dirCb/
 fileAa
-symlink-to-dirCb@
-
-./dir08a/dirCb:
-fileCba
-fileCbb
 
 ./dir08b:
 fileAa
@@ -221,51 +214,6 @@ file19c1
 
 ./dir19/dir19a3:
 file19b1
-
-./dir20:
-dir1/
-dir2/
-dir3/
-dir4/
-dirx/
-diry/
-file1
-file2
-file3
-file4
-filex
-filey
-s_dir1
-s_dir2@
-s_dir3@
-s_dir4/
-s_file1
-s_file2@
-s_file3@
-s_file4/
-
-./dir20/dir1:
-file_
-
-./dir20/dir2:
-file_
-
-./dir20/dir3:
-file_
-
-./dir20/dir4:
-file_
-
-./dir20/dirx:
-
-./dir20/diry:
-file_
-
-./dir20/s_dir4:
-file_
-
-./dir20/s_file4:
-file_
 EOF
 }
 
@@ -298,8 +246,6 @@ f: ./dir07c/fileAa: dirA/fileAa
 f: ./dir07d/fileAa: dirA/fileAa
 f: ./dir07e/fileAa: dirA/fileAa
 f: ./dir07f/fileAa: dirA/fileAa
-f: ./dir08a/dirCb/fileCba: dirCa/dirCb/fileCba
-f: ./dir08a/dirCb/fileCbb: dirCa/dirCb/fileCbb
 f: ./dir08a/fileAa: dirA/fileAa
 f: ./dir08b/fileAa: dirA/fileAa
 f: ./dir09/fileAa: dirA/fileAa
@@ -336,21 +282,6 @@ f: ./dir19/dir19a3/file19b1: new
 f: ./dir19/file19a1: old
 f: ./dir19/file19a2: new
 f: ./dir19/file19a3: new
-f: ./dir20/dir1/file_: dir1/file_
-f: ./dir20/dir2/file_: dir2/file_
-f: ./dir20/dir3/file_: dir3/file_
-f: ./dir20/dir4/file_: dir4/file_
-f: ./dir20/diry/file_: diry/file_
-f: ./dir20/file1: file1
-f: ./dir20/file2: file2
-f: ./dir20/file3: file3
-f: ./dir20/file4: file4
-f: ./dir20/filex: new
-f: ./dir20/filey: new
-f: ./dir20/s_dir1: new
-f: ./dir20/s_dir4/file_: s_dir4/file_
-f: ./dir20/s_file1: new
-f: ./dir20/s_file4/file_: s_file4/file_
 f: ./file1a: fileA
 f: ./file1b: fileA
 f: ./file2: fileB
@@ -367,14 +298,9 @@ EOF
 
     diff -u - <(ch-run --cd /test "$ch_img" \
                 -- find . -type l -printf '%y: %p -> %l\n' | sort) <<EOF
-l: ./dir08a/symlink-to-dirCb -> dirCb
 l: ./dir14/symlink-to-fileDa -> fileDa
 l: ./dir15/symlink-to-fileDa -> fileDa
 l: ./dir16/symlink-to-dirEb -> dirEb
-l: ./dir20/s_dir2 -> filey
-l: ./dir20/s_dir3 -> diry
-l: ./dir20/s_file2 -> filey
-l: ./dir20/s_file3 -> diry
 l: ./symlink-to-dir01c -> dir01c
 l: ./symlink-to-dir01d -> /test/dir01d
 l: ./symlink-to-dir01e -> dir01e