Merge branch 'develop' into dlpx/pr/sanjipta-delphix/5a72f2ec-b232-4f…

…b4-ac6e-4216605f697a

README.md

@@ -13,9 +13,9 @@ EC2, Microsoft Azure, VMware, OpenStack).
 
 Run this command on "dlpxdc.co" to create the VM used to do the build:
 
-    $ dc clone-latest --size COMPUTE_LARGE bootstrap-18-04 $USER-bootstrap
+    $ dc clone-latest --size COMPUTE_LARGE dlpx-internal-buildserver-develop $USER-bootstrap
 
-Log into that VM using the "ubuntu" user, and run these commands:
+Log into that VM using the "delphix" user, and run these commands:
 
     $ git clone https://github.com/delphix/appliance-build.git
     $ cd appliance-build
@@ -52,15 +52,15 @@ images.
 
 ### Step 1: Create Docker Host using DCenter on AWS
 
-Delphix maintains the "bootstrap-18-04" group in DCenter on AWS that
+Delphix maintains the "bootstrap-20-04" group in DCenter on AWS that
 fulfills the required build dependencies previously described. Thus, the
 first step is to use this group to create the host that will be used to
 execute the build. This can be done as usual, using "dc clone-latest".
 
 Example commands running on "dlpxdc.co":
 
     $ export DLPX_DC_INSTANCE_PUB_KEY=~/.ssh/id_rsa.pub
-    $ dc clone-latest --size COMPUTE_LARGE bootstrap-18-04 ps-build
+    $ dc clone-latest --size COMPUTE_LARGE bootstrap-20-04 ps-build
 
 Use the "ubuntu" user to log in to the VM after it's cloned; all of the
 following steps assume their being run on the cloned VM.

live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary

@@ -27,6 +27,23 @@ die() {
 	exit 1
 }
 
+#
+# usage: retry <attempts> <delay> command arg1 arg2 ...
+#
+retry() {
+	attempts=$1
+	shift
+	delay=$1
+	shift
+
+	for attempt in $(seq "${attempts}"); do
+		"$@" && break
+		[[ $attempt == "$attempts" ]] && die "Too many failed attempts, aborting."
+		echo "Attempt $attempt failed, trying again after a small nap."
+		sleep "$delay"
+	done
+}
+
 #
 # The root filesystem container needs to have the appliance version
 # embedded in it as a dataset property, thus if this value was not
@@ -43,15 +60,22 @@ die() {
 # We want to use different sized rpool depending on if we're building a
 # disk image meant for internal use, or external (i.e. customer) use.
 #
-# The only exception to this is our "dcenter" variant. While that
-# variant is only used internally, we use it in a way that more
-# resembles our external variants, so we want the rpool size for
-# the dcenter images to match our external images.
+# One exception to this is our "dcenter" variant. While that variant is
+# only used internally, we use it in a way that more resembles our
+# external variants, so we want the rpool size for the dcenter images to
+# match our external images.
+#
+# Another exception is our "buildserver" variant. We use those images to
+# generate our appliance images, and upgrade images, which consume a lot
+# of space during the build process; thus, we need a larger rpool.
 #
 case "$APPLIANCE_VARIANT" in
 external-* | internal-dcenter)
 	RAW_DISK_SIZE_GB=127
 	;;
+internal-buildserver)
+	RAW_DISK_SIZE_GB=256
+	;;
 internal-*)
 	RAW_DISK_SIZE_GB=70
 	;;
@@ -122,38 +146,39 @@ zpool create -d \
 	-O mountpoint=none \
 	-O compression=on \
 	-R "$DIRECTORY" \
+	-t "$FSNAME" \
 	rpool "/dev/mapper/${LOOPNAME}p1"
 
 zfs create \
 	-o canmount=off \
 	-o mountpoint=none \
-	"rpool/ROOT"
+	"$FSNAME/ROOT"
 
 zfs create \
 	-o canmount=off \
 	-o mountpoint=none \
 	-o "com.delphix:initial-version=$DELPHIX_APPLIANCE_VERSION" \
 	-o "com.delphix:current-version=$DELPHIX_APPLIANCE_VERSION" \
-	"rpool/ROOT/$FSNAME"
+	"$FSNAME/ROOT/$FSNAME"
 
 if [[ -n "$DELPHIX_HOTFIX_VERSION" ]]; then
 	zfs set \
 		"com.delphix:hotfix-version=$DELPHIX_HOTFIX_VERSION" \
-		"rpool/ROOT/$FSNAME"
+		"$FSNAME/ROOT/$FSNAME"
 fi
 
 if [[ -n "$DELPHIX_MINIMUM_VERSION" ]]; then
 	zfs set \
 		"com.delphix:minimum-version=$DELPHIX_MINIMUM_VERSION" \
-		"rpool/ROOT/$FSNAME"
+		"$FSNAME/ROOT/$FSNAME"
 fi
 
 zfs create \
 	-o canmount=noauto \
 	-o mountpoint=/ \
-	"rpool/ROOT/$FSNAME/root"
+	"$FSNAME/ROOT/$FSNAME/root"
 
-zfs mount "rpool/ROOT/$FSNAME/root"
+zfs mount "$FSNAME/ROOT/$FSNAME/root"
 
 #
 # We are later going to recursively bind mount /proc/, /sys/, and /dev/
@@ -188,15 +213,15 @@ mount --make-slave "$DIRECTORY"
 
 zfs create \
 	-o mountpoint=legacy \
-	"rpool/ROOT/$FSNAME/home"
+	"$FSNAME/ROOT/$FSNAME/home"
 
 zfs create \
 	-o mountpoint=legacy \
-	"rpool/ROOT/$FSNAME/data"
+	"$FSNAME/ROOT/$FSNAME/data"
 
 zfs create \
 	-o mountpoint=legacy \
-	"rpool/ROOT/$FSNAME/log"
+	"$FSNAME/ROOT/$FSNAME/log"
 
 #
 # Initialize the grub dataset. This dataset will be used to contain all
@@ -217,7 +242,7 @@ zfs create \
 #
 zfs create \
 	-o mountpoint=legacy \
-	rpool/grub
+	"$FSNAME/grub"
 
 #
 # Initialize the crashdump dataset. This is used to store core files
@@ -227,8 +252,8 @@ zfs create \
 #
 zfs create \
 	-o mountpoint=legacy \
-	-o quota="$(echo "$(zpool list -Hpo size rpool) / 2" | bc)b" \
-	rpool/crashdump
+	-o quota="$(echo "$(zpool list -Hpo size "$FSNAME") / 2" | bc)b" \
+	"$FSNAME/crashdump"
 
 #
 # Since these datasets use "legacy" for their mountpoints, we need to
@@ -237,16 +262,16 @@ zfs create \
 # these mounts.
 #
 mkdir -p "$DIRECTORY/export/home"
-mount -t zfs "rpool/ROOT/$FSNAME/home" "$DIRECTORY/export/home"
+mount -t zfs "$FSNAME/ROOT/$FSNAME/home" "$DIRECTORY/export/home"
 
 mkdir -p "$DIRECTORY/var/delphix"
-mount -t zfs "rpool/ROOT/$FSNAME/data" "$DIRECTORY/var/delphix"
+mount -t zfs "$FSNAME/ROOT/$FSNAME/data" "$DIRECTORY/var/delphix"
 
 mkdir -p "$DIRECTORY/var/log"
-mount -t zfs "rpool/ROOT/$FSNAME/log" "$DIRECTORY/var/log"
+mount -t zfs "$FSNAME/ROOT/$FSNAME/log" "$DIRECTORY/var/log"
 
 mkdir -p "/var/crash"
-mount -t zfs "rpool/crashdump" "/var/crash"
+mount -t zfs "$FSNAME/crashdump" "/var/crash"
 
 #
 # Populate the root filesystem with the contents of the "binary" directory
@@ -299,24 +324,19 @@ done
 # We need to use the dedicated grub dataset when running "grub-install"
 # and "grub-mkconfig", so we need to mount this dataset first.
 #
-chroot "$DIRECTORY" mount -t zfs rpool/grub /mnt
+chroot "$DIRECTORY" mount -t zfs "$FSNAME/grub" /mnt
 chroot "$DIRECTORY" grub-install --root-directory=/mnt "/dev/$LOOPNAME"
 chroot "$DIRECTORY" grub-mkconfig -o /mnt/boot/grub/grub.cfg
 chroot "$DIRECTORY" umount /mnt
 
 for dir in /dev /proc /sys; do
-	for attempt in {1..5}; do
-		umount -R "${DIRECTORY}${dir}" && break
-		[[ "$attempt" == 5 ]] && die "Too many failed attempts, aborting."
-		echo "Attempt $attempt failed, trying again after a small nap."
-		sleep 10
-	done
+	retry 5 10 umount -R "${DIRECTORY}${dir}"
 done
 
 umount "$DIRECTORY/var/log"
 umount "$DIRECTORY/var/delphix"
 umount "$DIRECTORY/export/home"
 umount "/var/crash"
-zfs umount "rpool/ROOT/$FSNAME/root"
-zpool export rpool
+retry 5 10 zfs umount "$FSNAME/ROOT/$FSNAME/root"
+retry 5 10 zpool export "$FSNAME"
 kpartx -d "$ARTIFACT_NAME.img"

live-build/config/hooks/vm-artifacts/92-ova-machine-image.binary

@@ -47,8 +47,7 @@ esac
 
 DATE=$(date --utc --iso-8601=seconds)
 
-VMDK_FILESIZE=$(qemu-img info --output=json "$ARTIFACT_NAME.vmdk" |
-	jq '.["actual-size"]')
+VMDK_FILESIZE=$(stat -c %s "$ARTIFACT_NAME.vmdk")
 VMDK_CAPACITY=$(qemu-img info --output=json "$ARTIFACT_NAME.vmdk" |
 	jq '.["virtual-size"]')
 

live-build/config/hooks/vm-artifacts/template.ovf

@@ -42,7 +42,7 @@
         <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
         <vssd:InstanceID>0</vssd:InstanceID>
         <vssd:VirtualSystemIdentifier>@@VM_NAME@@</vssd:VirtualSystemIdentifier>
-        <vssd:VirtualSystemType>vmx-11</vssd:VirtualSystemType>
+        <vssd:VirtualSystemType>vmx-13</vssd:VirtualSystemType>
       </System>
       <Item>
         <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
@@ -151,7 +151,7 @@
     </VirtualHardwareSection>
     <AnnotationSection ovf:required="false">
       <Info>A human-readable annotation</Info>
-      <Annotation>Delphix Appliance, VM Hardware Version 11</Annotation>
+      <Annotation>Delphix Appliance, VM Hardware Version 13</Annotation>
     </AnnotationSection>
   </VirtualSystem>
 </Envelope>

live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml

@@ -0,0 +1,30 @@
+#
+# Copyright 2022 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+- apt:
+    name:
+      - adoptopenjdk-java8-jdk
+      - curl
+      - docker.io
+      - git
+      - gnupg
+    state: present
+
+- user:
+    name: delphix
+    groups: docker
+    append: true

live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml

@@ -15,21 +15,13 @@
 #
 
 ---
-#
-# We are wrapping this in a retry block because the Gitlab server sometimes
-# hangs up unexpectedly. We haven't seen this with GitHub, so we should
-# probably remove the retry once we switch to GitHub.
-#
 - git:
-    repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git'
-    version: master
+    repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dcenter-gate.git"
+    version: main
     dest: /opt/dcenter/lib/dcenter-gate
     accept_hostkey: yes
     update: no
-  retries: 3
-  delay: 30
-  register: result
-  until: result is not failed
+  when: lookup('env', 'GITHUB_TOKEN') != ''
 
 - alternatives:
     name: java

live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml

@@ -0,0 +1,24 @@
+#
+# Copyright 2022 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+- git:
+    repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/devops-gate.git"
+    version: main
+    dest: /opt/devops-gate
+    accept_hostkey: yes
+    update: no
+  when: lookup('env', 'GITHUB_TOKEN') != ''

live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml

@@ -23,26 +23,14 @@
   retries: 3
   delay: 60
 
-#
-# We are wrapping this in a retry block because the Gitlab server sometimes
-# hangs up unexpectedly. We haven't seen this with GitHub, so we should
-# probably remove the retry once we switch to GitHub.
-#
 - git:
-    repo: "{{ item.repo }}"
+    repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dms-core-gate.git"
     dest:
-      "/export/home/delphix/{{ item.dest }}"
-    version: "{{ item.version }}"
+      "/export/home/delphix/dms-core-gate"
+    version: "develop"
     accept_hostkey: yes
     update: no
-  with_items:
-    - { repo: 'https://gitlab.delphix.com/masking/dms-core-gate.git',
-        version: master,
-        dest: dms-core-gate }
-  retries: 3
-  delay: 30
-  register: result
-  until: result is not failed
+  when: lookup('env', 'GITHUB_TOKEN') != ''
 
 - file:
     path: "/export/home/delphix/{{ item }}"

live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml

@@ -35,6 +35,7 @@
     uid: 65433
     group: staff
     groups: root
+    append: true
     shell: /bin/bash
     create_home: yes
     comment: Delphix User
@@ -68,3 +69,18 @@
   delay: 30
   register: result
   until: result is succeeded
+
+- shell: ls /lib/modules
+  register: kernel_versions
+
+#
+# In order to generate BTF data when building the ZFS kernel module
+# through git-zfs-make we need to provide a symlink of our vmlinux
+# (with debug info) in the kernel header source directory which is
+# referenced during the kernel module's compilation.
+#
+- ansible.builtin.file:
+    src: '/usr/lib/debug/boot/vmlinux-{{ item }}'
+    dest: '/usr/src/linux-headers-{{ item }}/vmlinux'
+    state: link
+  loop: '{{ kernel_versions.stdout_lines }}'