Merge branch 'development'

GoogleCloudPlatform · Jul 29, 2019 · fa80cdb · fa80cdb
2 parents 5d9e07f + 466dc6e
commit fa80cdb
Show file tree

Hide file tree

Showing 94 changed files with 1,956 additions and 1,039 deletions.
diff --git a/README.md b/README.md
@@ -49,6 +49,7 @@ Ubuntu 16.04 | deb          | 3.5 or 2.7     | systemd
 Ubuntu 18.04 | deb          | 3.6            | systemd
 Ubuntu 19.04 | deb          | 3.7            | systemd
 Debian 9     | deb          | 3.5 or 2.7     | systemd
+Debian 10    | deb          | 3.7            | systemd
 
 We build the following packages for the Linux guest environment.
 
@@ -75,6 +76,12 @@ packaging directory for more details.
 
 #### Version Updates
 
+Versions are described as 1:YYYYMMDD.NN-gN, meaning epoch 1 to denote from a
+distro maintained package which will be 0, a date string formatted as year,
+month, day, an incrementing minor release, and gN representing the Google
+package release. Debian, Ubuntu, and SUSE maintain distro packages which may be
+out of date, have different versioning, or naming.
+
 The method for making version updates differs by package.
 
 * All packages need the `VERSION` variable set in the `setup_{deb,rpm}.sh` build
@@ -102,11 +109,13 @@ Add the public repo key to your system:
 curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
 ```
 
-Add a source list file `/etc/apt/sources.list.d/google-cloud.list`:
+Add a source list file `/etc/apt/sources.list.d/google-cloud.list` and change
+`DIST` to either `stretch` for Debian 9 or `buster` for Debian 10:
 ```
+DIST=stretch
 sudo tee /etc/apt/sources.list.d/google-cloud.list << EOM
-deb http://packages.cloud.google.com/apt google-compute-engine-stretch-stable main
-deb http://packages.cloud.google.com/apt google-cloud-packages-archive-keyring-stretch main
+deb http://packages.cloud.google.com/apt google-compute-engine-${DIST}-stable main
+deb http://packages.cloud.google.com/apt google-cloud-packages-archive-keyring-${DIST} main
 EOM
 ```
 
@@ -119,8 +128,8 @@ You are then able to install any of the packages from this repo.
 
 **For RedHat based distributions, run the following commands as root:**
 
-Add the yum repo to a repo file `/etc/yum.repos.d/google-cloud.repo` for either
-EL6 or EL7. Change `DIST` to either 6 or 7 respectively:
+Add the yum repo to a repo file `/etc/yum.repos.d/google-cloud.repo` for EL6,
+EL7, or EL8. Change `DIST` to either 6, 7, or 8 respectively:
 ```
 DIST=7
 tee /etc/yum.repos.d/google-cloud.repo << EOM

diff --git a/daisy_workflows/build_deb_packages.sh b/daisy_workflows/build_deb_packages.sh
@@ -56,7 +56,13 @@ done
 
 # Copy the deb and dsc files to the output.
 cd "${workdir}/output"
-gsutil cp *.dsc *.deb ${OUTPUT}
+# For Debian 10 right now, only copy OS Login package.
+if [[ "$(cut -d. -f1 </etc/debian_version)" == "10" ]]; then
+  gsutil cp google-compute-engine-oslogin*.* ${OUTPUT}
+else
+  gsutil cp *.dsc *.deb ${OUTPUT}
+fi
+
 if [ $? -ne 0 ]; then
   echo "BuildFailed: copying to ${OUTPUT} failed."
   exit 1

diff --git a/daisy_workflows/build_debian.wf.json b/daisy_workflows/build_debian.wf.json
@@ -20,19 +20,39 @@
     "setup-disk": {
       "CreateDisks": [
         {
-          "Name": "disk-deb-build",
+          "Name": "disk-deb9-build",
           "SourceImage": "projects/debian-cloud/global/images/family/debian-9",
           "SizeGb": "10",
           "Type": "pd-ssd"
+        },
+        {
+          "Name": "disk-deb10-build",
+          "SourceImage": "projects/debian-cloud/global/images/family/debian-10",
+          "SizeGb": "10",
+          "Type": "pd-ssd"
         }
       ]
     },
     "package-build": {
       "CreateInstances": [
         {
-          "Name": "inst-deb-build",
+          "Name": "inst-deb9-build",
+          "Disks": [
+            {"Source": "disk-deb9-build"}
+          ],
+          "MachineType": "n1-standard-2",
+          "Metadata": {
+            "github_branch": "${github_branch}",
+            "github_repo": "${github_repo}",
+            "output_path": "${output_path}"
+          },
+          "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
+          "StartupScript": "build_deb_packages.sh"
+        },
+        {
+          "Name": "inst-deb10-build",
           "Disks": [
-            {"Source": "disk-deb-build"}
+            {"Source": "disk-deb10-build"}
           ],
           "MachineType": "n1-standard-2",
           "Metadata": {
@@ -48,7 +68,15 @@
     "wait-for-build": {
       "WaitForInstancesSignal": [
         {
-          "Name": "inst-deb-build",
+          "Name": "inst-deb9-build",
+          "SerialOutput": {
+            "Port": 1,
+            "SuccessMatch": "BuildSuccess:",
+            "FailureMatch": "BuildFailed:"
+          }
+        },
+        {
+          "Name": "inst-deb10-build",
           "SerialOutput": {
             "Port": 1,
             "SuccessMatch": "BuildSuccess:",

diff --git a/packages/gce-disk-expand/README.md b/packages/gce-disk-expand/README.md
@@ -1,53 +1,11 @@
-## gce-disk-expand package for CentOS/RHEL
+## gce-disk-expand package for CentOS/RHEL and Debian
 
-This package is intended to expand the root partition up to 2TB on a GCE VM. It
-consists of several scripts from other packages that for various reasons are not
-maintained by the distros; cloud-utils-growpart, dracut-modules-growroot, and an
-upstream version of the growpart script. See below for details. This package is
-included in GCE CentOS and RHEL images after v20160415.
+This package is intended to expand the root partition up to 2TB on a GCE VM
+without a GPT partition table and over 2TB on GPT partitioned UEFI enabled
+images.
 
-### Build the package
+This package has been tested on the following distros and versions.
 
-A script to build the package is provided for convenience.
-`./build_package.sh /OUTPUT_DIR` will yield an rpm and an srpm in the
-defined output directory for EL6 and EL7.
-
-### Package usage
-
-Install the gce-disk-expand package for your distro with yum and reboot:
-
-* CentOS/RHEL 6: `yum -y install /PATH_TO/gce-disk-expand-el6-VER-DATE.x86_64.rpm ; reboot`
-
-* CentOS/RHEL 7: `yum -y install /PATH_TO/gce-disk-expand-el7-VER-DATE.x86_64.rpm ; reboot`
-
-Your root partition will now be expanded to the full size of your disk up to
-2TB.
-
-#### Growpart
-
-The growpart script used in this package is not from the 0.27
-cloud-utils-growpart package. There are bugs in that version of growpart,
-primarily a bug that doesn't respect 2TB disk partitions on MBR disks. This
-means that if you had a disk that was 2.1TB, you would end up with a 0.1TB
-partition instead of 2TB partition. Obviously, this is bad. The upstream
-[cloud-utils 257 release](http://bazaar.launchpad.net/~cloud-utils-dev/cloud-utils/trunk/tarball/257)
-fixes this bug and others while not introducing more bugs and dependencies in
-further upstream releases of this script. It is therefore not recommended that
-you try to use the [0.27 growpart package](http://rpmfind.net/linux/RPM/epel/6/x86_64/cloud-utils-growpart-0.27-10.el6.x86_64.html)
-from the EPEL repo.
-
-#### Dracut module for EL6
-
-The dracut module is taken from the [dracut-modules-growroot](http://rpmfind.net/linux/RPM/epel/6/x86_64/dracut-modules-growroot-0.20-2.el6.noarch.html)
-package in the EPEL 6 repo. The dracut module allows the partition table to be
-expanded on boot before / is mounted and prevents an additional reboot in CentOS
-and RHEL 6. CentOS and RHEL 7 support live partition resizing by the kernel and
-do not need the dracut module.
-
-#### The expand-root script
-
-The expand-root script is derived from the
-[bootstrap-vz version](https://github.com/andsens/bootstrap-vz/blob/c682dab6ec51767b6e529c1589c5630f6295953a/bootstrapvz/common/assets/init.d/expand-root)
-of this script used for Debian instances. Essentially, it just calls the proper
-filesystem expansion utility to live resize the filesystem on first boot. The
-root partition is hard coded as /dev/sda1 in this script.
+* RHEL/CentOS 7.4+
+* RHEL 8+
+* Debian 10+
diff --git a/packages/gce-disk-expand/packaging/debian/changelog b/packages/gce-disk-expand/packaging/debian/changelog
@@ -1,3 +1,9 @@
+gce-disk-expand (1:20190708.00-g1) stable; urgency=medium
+
+  * Update disk expand to fix bugs with GPT.
+
+ -- Google Cloud Team <[email protected]>  Mon, 08 Jul 2019 10:20:47 -0700
+
 gce-disk-expand (2.0.0-1) stable; urgency=low
 
   * Initial debian package import.

diff --git a/packages/gce-disk-expand/packaging/debian/control b/packages/gce-disk-expand/packaging/debian/control
@@ -2,8 +2,7 @@ Source: gce-disk-expand
 Section: admin
 Priority: optional
 Maintainer: Google Cloud Team <[email protected]>
-Build-Depends: debhelper (>= 9),
-               dh-systemd (>= 1.5),
+Build-Depends: debhelper (>= 10)
 Standards-Version: 3.9.8
 Homepage: https://github.com/GoogleCloudPlatform/compute-image-packages
 

diff --git a/packages/gce-disk-expand/packaging/debian/install b/packages/gce-disk-expand/packaging/debian/install
@@ -1 +1,2 @@
 src/usr/share/initramfs-tools usr/share/
+src/expandfs-lib.sh usr/share/initramfs-tools/scripts/
diff --git a/packages/gce-disk-expand/packaging/gce-disk-expand.spec b/packages/gce-disk-expand/packaging/gce-disk-expand.spec
@@ -13,8 +13,9 @@
 # limitations under the License.
 Name: gce-disk-expand
 Summary: Google Compute Engine root disk expansion module
-Version: 2.0.0
-Release: 1
+Epoch: 1
+Version: %{_version}
+Release: g1
 License: Apache Software License
 Group: System Environment/Base
 URL: https://github.com/GoogleCloudPlatform/compute-image-packages

diff --git a/packages/gce-disk-expand/packaging/setup_deb.sh b/packages/gce-disk-expand/packaging/setup_deb.sh
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="gce-disk-expand"
-VERSION="2.0.0"
+VERSION="20190708.00"
 
 working_dir=${PWD}
 if [[ $(basename "$working_dir") != $NAME ]]; then

diff --git a/packages/gce-disk-expand/packaging/setup_rpm.sh b/packages/gce-disk-expand/packaging/setup_rpm.sh
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="gce-disk-expand"
-VERSION="2.0.0"
+VERSION="20190708.00"
 
 rpm_working_dir=/tmp/rpmpackage/
 working_dir=${PWD}
@@ -30,6 +30,8 @@ rm -rf ${rpm_working_dir}
 mkdir -p ${rpm_working_dir}/{SOURCES,SPECS}
 cp packaging/${NAME}.spec ${rpm_working_dir}/SPECS/
 
-tar czvf ${rpm_working_dir}/SOURCES/${NAME}_${VERSION}.orig.tar.gz  --exclude .git --exclude packaging --transform "s/^\./${NAME}-${VERSION}/" .
+tar czvf ${rpm_working_dir}/SOURCES/${NAME}_${VERSION}.orig.tar.gz  \
+  --exclude .git --exclude packaging --transform "s/^\./${NAME}-${VERSION}/" .
 
-rpmbuild --define "_topdir ${rpm_working_dir}/" -ba ${rpm_working_dir}/SPECS/${NAME}.spec
+rpmbuild --define "_topdir ${rpm_working_dir}/" --define "_version ${VERSION}" \
+  -ba ${rpm_working_dir}/SPECS/${NAME}.spec
diff --git a/packages/gce-disk-expand/src/expandfs-lib.sh b/packages/gce-disk-expand/src/expandfs-lib.sh
@@ -67,14 +67,15 @@ parted_fix_gpt() {
   local disk="$1"
   [ -z "$disk" ] && return
 
-  if parted -sm "$rootdisk" print 2>&1 | grep "fix the GPT"; then
+  if parted -sm "$disk" print 2>&1 | grep -q "fix the GPT"; then
     # Running parted prompts the user to fix this condition, but only does so in
     # the interactive exception handler. In order to pass input we must use the
     # hidden triple-dash flag and pass both print and Fix arguments. `print`
     # alone will not perform the fix, but `Fix` alone will fail the argument
     # parser.
-    parted -m ---pretend-input-tty "$rootdisk" print Fix
-    if parted -sm "$rootdisk" print 2>&1 | grep "fix the GPT"; then
+    parted -m ---pretend-input-tty "$disk" print Fix >/dev/null 2>&1 </dev/null
+    parted -m ---pretend-input-tty "$disk" print Fix >/dev/null 2>&1 </dev/null
+    if parted -sm "$disk" print 2>&1 | grep -q "fix the GPT"; then
       echo "Failed to fix the GPT."
       return 1
     fi
@@ -110,13 +111,15 @@ parted_needresize() {
     return 1
   fi
 
-  if ! echo -e "$out" | sed '$!d' | grep -q "^${partnum}:"; then
+  udevadm settle
+
+  if ! printf "$out" | sed '$!d' | grep -q "^${partnum}:"; then
     echo "Root partition is not final partition on disk. Not resizing."
     return 1
   fi
 
-  disksize=$(echo -e "$out" | grep "^${disk}" | cut -d: -f2)
-  partend=$(echo -e "$out" | sed '$!d' | cut -d: -f4)
+  disksize=$(printf "$out" | grep "^${disk}" | cut -d: -f2)
+  partend=$(printf "$out" | sed '$!d' | cut -d: -f4)
   [ -n "$disksize" -a -n "$partend" ] || return 1
 
   disksize=${disksize%%B}
@@ -138,52 +141,3 @@ parted_resizepart() {
   fi
   udevadm settle
 }
-
-# Resizes partition by deleting and recreating with end position.
-# This is a subshell function to safeguard against modifications of IFS.
-parted_resize_mkpart() (
-  local disk="$1" partnum="$2"
-  [ -z "$disk" -o -z "$partnum" ] && return
-
-  local partnum="" partbegin="" partend="" partsize=""
-  local fstype="" partname="" flags="" temp=""
-
-  # $ parted -sm /dev/sda unit b print
-  # BYT;
-  # /dev/sda:18253611008B:scsi:512:4096:msdos:Google PersistentDisk:;
-  # 1:2097152B:18252611583B:18250514432B:ext4::boot;
-  #
-  if ! out=$(parted -sm "$disk" unit b print 2>&1); then
-    echo "Unable to get partition info."
-    return 1
-  fi
-
-  temp=/tmp/my_temp
-  echo -e "$out" | sed '$!d' > $temp
-  IFS=: read partnum partbegin partend partsize fstype partname flags < $temp
-  rm $temp
-
-  if ! out=$(parted -sm "$disk" rm $partnum 2>&1); then
-    echo "Failed to delete partition: ${out}"
-    return 1
-  fi
-
-  if ! out=$(parted -sm "$disk" -- mkpart pri $fstype $partbegin -1 2>&1); then
-    echo "Failed to recreate original partition: ${out}"
-    echo "Trying to create with original parameters."
-    if ! out=$(parted -sm "$disk" mkpart pri $fstype $partbegin $partend 2>&1); then
-      echo "Failed to recreate original partition: ${out}"
-      return 1
-    fi
-  fi
-
-  flags=${flags%%;}
-  IFS=,
-  for flag in $flags; do
-    if ! out=$(parted -sm "$disk" set $partnum $flag on 2>&1); then
-      echo "Failed to set \"$flag\" on ${disk} part ${partnum}: ${out}"
-      return 1
-    fi
-  done
-  udevadm settle
-)