From 8a397f3cff99eabf552827e3cd477f496df55ab4 Mon Sep 17 00:00:00 2001 From: Garrett Rodriguez Date: Fri, 4 Oct 2019 15:18:25 -0700 Subject: [PATCH 001/197] [Backport of LB-209 to 6.0/release] Want to consume new mirror for production appliance build --- branch.config | 13 +++++ scripts/build-ancillary-repository.sh | 13 +++-- scripts/docker-run.sh | 24 ++++++++ scripts/run-live-build.sh | 83 +++++++++++++++++++-------- 4 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 branch.config diff --git a/branch.config b/branch.config new file mode 100644 index 00000000..8b13b1e8 --- /dev/null +++ b/branch.config @@ -0,0 +1,13 @@ +# +# Copyright (c) 2019 by Delphix. All rights reserved. +# + +# +# The "BRANCH" parameter tracks the upstream branch of appliance-build. It is +# used to determine which branch of the linux package mirror will be used for +# the build if UPSTREAM_PRODUCT_BRANCH is not set. UPSTREAM_PRODUCT_BRANCH is +# set when appliance build is built by the appliance-build Jenkins job. The +# UPSTREAM_BRANCH parameter should be updated by the release lead on branching +# + +UPSTREAM_BRANCH="6.0/release" diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 047f1d74..133871fc 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -142,27 +142,30 @@ function build_ancillary_repository() { # environment variables, and the script will work as expected. # -upstream_branch="${UPSTREAM_PRODUCT_BRANCH:-master}" +if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH is not set." + exit 1 +fi AWS_S3_URI_VIRTUALIZATION=$(resolve_s3_uri \ "$AWS_S3_URI_VIRTUALIZATION" \ "$AWS_S3_PREFIX_VIRTUALIZATION" \ - "dlpx-app-gate/${upstream_branch}/build-package/post-push/latest") + "dlpx-app-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_MASKING=$(resolve_s3_uri \ "$AWS_S3_URI_MASKING" \ "$AWS_S3_PREFIX_MASKING" \ - "dms-core-gate/${upstream_branch}/build-package/post-push/latest") + "dms-core-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_USERLAND_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_USERLAND_PKGS" \ "$AWS_S3_PREFIX_USERLAND_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/userland/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/userland/post-push/latest") AWS_S3_URI_KERNEL_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_KERNEL_PKGS" \ "$AWS_S3_PREFIX_KERNEL_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/kernel/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/kernel/post-push/latest") # # All package files will be placed into this temporary directory, such diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index a4a240a0..733200c4 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -38,6 +38,29 @@ else DOCKER_RUN="docker run" fi +# +# Set UPSTREAM_BRANCH. This will determine which version of the linux package +# mirror is used. +# +if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + echo "UPSTREAM_PRODUCT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ + "Ensure branch.config is properly formatted with e.g." \ + "UPSTREAM_BRANCH=\"\"" + exit 1 + fi + echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." +else + UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" +fi +echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + $DOCKER_RUN --rm \ --privileged \ --network host \ @@ -64,6 +87,7 @@ $DOCKER_RUN --rm \ --env DELPHIX_SIGNATURE_VERSIONS \ --env DELPHIX_UPGRADE_MINIMUM_VERSION \ --env DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION \ + --env UPSTREAM_BRANCH="$UPSTREAM_BRANCH" \ --volume "$TOP:/opt/appliance-build" \ --workdir "/opt/appliance-build" \ appliance-build "$@" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 5783877a..4a01537b 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -127,37 +127,72 @@ while ! curl --output /dev/null --silent --head --fail \ sleep 1 done -set -o errexit +pkg_mirror_secondary='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_SECONDARY" ]]; then - sed "s|@@URL@@|$DELPHIX_PACKAGE_MIRROR_SECONDARY|" \ - config/archives/delphix-secondary-mirror.list + pkg_mirror_secondary="$DELPHIX_PACKAGE_MIRROR_SECONDARY" +else + # + # If no secondary package mirror is provided, then pull in the latest + # mirror dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_secondary=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No URL found for PPA packages at ${source_url}." + exit 1 + fi + + # The mirror hosts secondary packages in the "ppas" subdirectory. + pkg_mirror_secondary+="ppas" fi +sed "s|@@URL@@|$pkg_mirror_secondary|" \ + config/archives/delphix-secondary-mirror.list + +pkg_mirror_main='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then - lb config \ - --parent-mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" + pkg_mirror_main="$DELPHIX_PACKAGE_MIRROR_MAIN" else - lb config + # + # If no main package mirror is provided, then pull in the latest mirror + # dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_main=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No mirror URL found for ubuntu archive packages at ${source_url}." + exit 1 + fi + + # + # The internal mirror hosts the primary ubuntu package repository in the + # "ubuntu" subdirectory. + # + pkg_mirror_main+="ubuntu" fi +set -o errexit + +lb config \ + --parent-mirror-bootstrap "$pkg_mirror_main" \ + --parent-mirror-chroot "$pkg_mirror_main" \ + --parent-mirror-chroot-security "$pkg_mirror_main" \ + --parent-mirror-chroot-volatile "$pkg_mirror_main" \ + --parent-mirror-chroot-backports "$pkg_mirror_main" \ + --parent-mirror-binary "$pkg_mirror_main" \ + --parent-mirror-binary-security "$pkg_mirror_main" \ + --parent-mirror-binary-volatile "$pkg_mirror_main" \ + --parent-mirror-binary-backports "$pkg_mirror_main" \ + --mirror-bootstrap "$pkg_mirror_main" \ + --mirror-chroot "$pkg_mirror_main" \ + --mirror-chroot-security "$pkg_mirror_main" \ + --mirror-chroot-volatile "$pkg_mirror_main" \ + --mirror-chroot-backports "$pkg_mirror_main" \ + --mirror-binary "$pkg_mirror_main" \ + --mirror-binary-security "$pkg_mirror_main" \ + --mirror-binary-volatile "$pkg_mirror_main" \ + --mirror-binary-backports "$pkg_mirror_main" lb build From 1586bfdb0d0d73463ae24630de4530e13cca3a0f Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 10 Oct 2019 11:15:49 -0700 Subject: [PATCH 002/197] [Backport to 6.0.0.0] Ensure that we are using secondary mirror (#371) --- live-build/auto/config | 18 ------------------ live-build/config/archives/ansible.key | Bin 1132 -> 0 bytes live-build/config/archives/ansible.list | 17 ----------------- .../archives/delphix-secondary-mirror.key | Bin 0 -> 2255 bytes .../archives/delphix-secondary-mirror.list.in | 3 ++- .../tasks/main.yml | 13 +------------ 6 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 live-build/config/archives/ansible.key delete mode 100644 live-build/config/archives/ansible.list create mode 100644 live-build/config/archives/delphix-secondary-mirror.key diff --git a/live-build/auto/config b/live-build/auto/config index e5c78fe1..2dc7b897 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -31,24 +31,6 @@ # delphix-platform. # lb config noauto \ - --parent-mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ --archive-areas "main restricted universe multiverse" \ --bootstrap-flavour minimal \ --chroot-filesystem none \ diff --git a/live-build/config/archives/ansible.key b/live-build/config/archives/ansible.key deleted file mode 100644 index 36ebbebd3a7b1c5e557b64eb1c2956ab2e8db650..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1132 zcmV-y1e5!j0u2OHmu)Zs5CF<8#y2sBavX?)67L*C?G(=zXq7e@f6Q6auif(;)LZod zJu=nD{xl;si4m1N>|mHxbs=?y#0s%cDFcoXWGuiOPPcBqgGtD(O&)chTnV3O_vTKp zpxjBtjm2l2l)Co$f!&#ZUZx+mr|if+XiX)imRGOaN{wFzF5qf*MMMWR-!r_9)GT8n z9-skmj@@r=PeO_JMTAEoMON^=mLX&{p9PrVXI@k~>Eol0;1Wto(EAo0j!ob9JLZv$ zz{J9ga|)i0iWfLtD7Lfx2pcwfa@C_G1MjY`Lz6(awYWneCUowKp!;(c|LRENDJuV@}J^R10+K>bBPj0 zCcgwPd7G4u#uSnVISiLz>kj2k1P55=&JfyHmd(KqdSL^A$;r(RK;Xi~)S07(S1`Hx zluz`k@CoV<@SfH`dGGH;tTOf>%sFH)iWJCW^83^D=M##Vb)#E2kuO`HOHZMJPPY`) z#rAzJVN>~#YhaCw6qRIee66;(wewmmFwLqfI#xM7Uyj4+Unp~Z9?dnFy^Ep&k9z1@*%FAsQAQg_q^ zZA|2Ax}@E=%BDtciOPD+c(7l!ho;1z9^v?+=mW{bPK7@|z`h3!+#5lH>mC;e zAZw{ca%yxWx*r-8+j=+B_0_#@aD^lrQGYcRci@}xrY_B1bATdXsNhK*{R`_5@Snvl z%@^X(#j_I0E(`LpP@0t-%adrgYf0nv{mkf^m#uEZhfVP0mH+ABM$$9YYGhc7uU_zb zLfwi01ixZuFJvRqgLiooRMNo7-bQbSnxUT8je#Ug{poZ{T)y9ck!Lmb=J$T22%i1(LzY!TWhePHBrdo)WGPY3)h z7C#hPORz)L!-36Ks;a^piJc|8(PTGnSad{%|NB2-0#?2=Ng~I(^p!6?WEWF|1MY|_ y zxKPcMYr<3pJMcg>$sX`efXd=13k#}T$VNIG=|Cy$?)&!gkYJ?_#3Pv#Ou;QI0gE!f z6uHR|e9TS1x6F=_^`V02gEP~%%Sl>oEtbaj2keKP*E6O}*OKv_hq$`g7UbAvSsKzZ9HkhdR-+tD8oJCZS~>gP;m@ z8~b1cNF0v@1)cza;cBs>Uk_OK)m2))n1!;zqC&#osjO9{WqlNG9;4_cF?mX@2!o zdQHE9yRU-ONb3L*0RRECBSK+wX>1@va&LDaJYr#UX>2ZIa&LD)WMyn{XlZyZV{dIf zi2_ap69EVSI2IuUnTFJG+#RXVPojPJv2@%0Ve))a1p->f=HCJv0|g5S2nPZN6$%Lm z3jzcd0s{d89svRufB*^!5V3UI{bBNaQ`Hd<{x?M!wg%G-9x%`ycOCrm)E@pP+}uI4 z<@^Ff;%=FKok6-NBFOUq=dT7|T^y++#Y(yVxH5Ao1_U9sEzIvw5(6ODb|oceZvwS0vvQY zI4iOqhbtbr7*5P+CI5nSG9IJ;GK8h{Csi9nkXF8N=y;U%Z(l+{dj_Q6y8E!^-g(YxJaGDYheD3%sL71vFP6c?rB83Q(X^XdjFB)^4(K0 z;DwjCtk$@e?zeHDmJEDe7CE%|;nH%9Mh-Y4&#>l(9gbX-idRhU8F>4mpsH06r5 z0ZWN8_p$?caNW*lxP9z9NpCzdUYJl3Tm{AAqb-AB<{jGI-1V~OLDJq7hGi&C@$F%p zIB~*E+Bz;Xo$?E~mG%y=7ir+Dleq#71X{=D-vJN+#nBb6;v%DPOJ`Bfem|)d)lry4 zd;D>G;<1`N7PF)jMU82Pk*N7Gn%=b+kHJye!ueFB5sO?BQH({o4^x-%>sdA_d9Ijc-5b3VsfFUv($?B)oz>m=kw|GDYJL~R30H%lA4&Epu_lb4*i>|E<|#-p95 zjV)Qz9*{S$wa2C#{{W-ho$~jT^W%xM!Y&UUJ`96@Jzm+p*JjVA-dRbqeAY>}*tu)D zCMi!N2!nw#RrAMxM^SL(3S)&VRmcU(=P4Ato*=*&9vHu6pkgaU z=7^1h1+t;p{P@tx=z%qAA3mj7&Ib?yrRjdd?gf^(dscqotumFD0dtF3l zQZ-yncp84%$blbM%wz8sPW1E!qr9E>E+lcPoG*v&tU!#<$Nk*bV3i@NOwGe7fJ~ql zsW0U_el`LO`0oyk_n91VHEoad23~EO01*KI0f_=O1Q-Db03a421eu1^aNHfK&`+X$ z__1``{bBNaQw0KA$L8Mx8w>yn2@tV#+x=nkd{c$a5B@OOK!M*BP5lWv=52whckhVec#b&Gp;6t8_hjPmex>PO&`WI_6LdkW_}b3W&!1lH|jyY@Ai;_3$f zwlcmd1R`TrR%;Hfit$T-t0qhxlK1eSl^Ke}@OYClx@r$%g|n518eFZD9VAIeyDJ%I za-y?Vu5o@j;|q%zTFO!&78C)k`e^JIbu|E=H|I&c?DstO*!mLlLoq)g-lE`BzQ?C% zsNzcb^BJhn;XRI#4f``ACbgMmn})~^ z;ddjYIk(4>=ML)t@hbV22E`r60rF03mf`eBVS0z|M~(i2}Zc zl5yXjXHr#KZcLMEcr4-Df(+-`TPh! dPuwM`LKr8?N literal 0 HcmV?d00001 diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 35816d28..3d42434f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,4 +14,5 @@ # limitations under the License. # -deb [trusted=yes] @@URL@@ bionic main +deb @@URL@@ bionic main multiverse universe +deb @@URL@@ bionic-updates main multiverse universe diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index 53894995..fcb8a962 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -51,23 +51,12 @@ # The virtualization package uses the /etc/issue file to store a # customer-supplied banner that is displayed prior to login. By default, # there should be no banner. Thus, we need to remove this file to -# acheive our desired behavior; Ubuntu ships with this file by default. +# achieve our desired behavior; Ubuntu ships with this file by default. # - file: path: /etc/issue state: absent -- apt_repository: - repo: "{{ item }}" - with_items: - - deb http://ddebs.ubuntu.com bionic main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-updates main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-proposed main restricted universe multiverse - retries: 3 - delay: 30 - register: result - until: result is succeeded - - shell: ls /lib/modules register: kernel_versions From 236fdaf1c0cdf3f6f4651d349ea52c0b9e7470eb Mon Sep 17 00:00:00 2001 From: grodr <39814162+grodr@users.noreply.github.com> Date: Thu, 10 Oct 2019 11:49:06 -0700 Subject: [PATCH 003/197] [Backport of LB-209 to 6.0/stage] Want to consume new mirror for production appliance build (#370) --- branch.config | 13 +++++ scripts/build-ancillary-repository.sh | 13 +++-- scripts/docker-run.sh | 24 ++++++++ scripts/run-live-build.sh | 83 +++++++++++++++++++-------- 4 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 branch.config diff --git a/branch.config b/branch.config new file mode 100644 index 00000000..9e572e67 --- /dev/null +++ b/branch.config @@ -0,0 +1,13 @@ +# +# Copyright (c) 2019 by Delphix. All rights reserved. +# + +# +# The "BRANCH" parameter tracks the upstream branch of appliance-build. It is +# used to determine which branch of the linux package mirror will be used for +# the build if UPSTREAM_PRODUCT_BRANCH is not set. UPSTREAM_PRODUCT_BRANCH is +# set when appliance build is built by the appliance-build Jenkins job. The +# UPSTREAM_BRANCH parameter should be updated by the release lead on branching +# + +UPSTREAM_BRANCH="6.0/stage" diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 047f1d74..133871fc 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -142,27 +142,30 @@ function build_ancillary_repository() { # environment variables, and the script will work as expected. # -upstream_branch="${UPSTREAM_PRODUCT_BRANCH:-master}" +if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH is not set." + exit 1 +fi AWS_S3_URI_VIRTUALIZATION=$(resolve_s3_uri \ "$AWS_S3_URI_VIRTUALIZATION" \ "$AWS_S3_PREFIX_VIRTUALIZATION" \ - "dlpx-app-gate/${upstream_branch}/build-package/post-push/latest") + "dlpx-app-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_MASKING=$(resolve_s3_uri \ "$AWS_S3_URI_MASKING" \ "$AWS_S3_PREFIX_MASKING" \ - "dms-core-gate/${upstream_branch}/build-package/post-push/latest") + "dms-core-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_USERLAND_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_USERLAND_PKGS" \ "$AWS_S3_PREFIX_USERLAND_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/userland/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/userland/post-push/latest") AWS_S3_URI_KERNEL_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_KERNEL_PKGS" \ "$AWS_S3_PREFIX_KERNEL_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/kernel/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/kernel/post-push/latest") # # All package files will be placed into this temporary directory, such diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index a4a240a0..733200c4 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -38,6 +38,29 @@ else DOCKER_RUN="docker run" fi +# +# Set UPSTREAM_BRANCH. This will determine which version of the linux package +# mirror is used. +# +if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + echo "UPSTREAM_PRODUCT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ + "Ensure branch.config is properly formatted with e.g." \ + "UPSTREAM_BRANCH=\"\"" + exit 1 + fi + echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." +else + UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" +fi +echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + $DOCKER_RUN --rm \ --privileged \ --network host \ @@ -64,6 +87,7 @@ $DOCKER_RUN --rm \ --env DELPHIX_SIGNATURE_VERSIONS \ --env DELPHIX_UPGRADE_MINIMUM_VERSION \ --env DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION \ + --env UPSTREAM_BRANCH="$UPSTREAM_BRANCH" \ --volume "$TOP:/opt/appliance-build" \ --workdir "/opt/appliance-build" \ appliance-build "$@" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 5783877a..4a01537b 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -127,37 +127,72 @@ while ! curl --output /dev/null --silent --head --fail \ sleep 1 done -set -o errexit +pkg_mirror_secondary='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_SECONDARY" ]]; then - sed "s|@@URL@@|$DELPHIX_PACKAGE_MIRROR_SECONDARY|" \ - config/archives/delphix-secondary-mirror.list + pkg_mirror_secondary="$DELPHIX_PACKAGE_MIRROR_SECONDARY" +else + # + # If no secondary package mirror is provided, then pull in the latest + # mirror dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_secondary=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No URL found for PPA packages at ${source_url}." + exit 1 + fi + + # The mirror hosts secondary packages in the "ppas" subdirectory. + pkg_mirror_secondary+="ppas" fi +sed "s|@@URL@@|$pkg_mirror_secondary|" \ + config/archives/delphix-secondary-mirror.list + +pkg_mirror_main='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then - lb config \ - --parent-mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" + pkg_mirror_main="$DELPHIX_PACKAGE_MIRROR_MAIN" else - lb config + # + # If no main package mirror is provided, then pull in the latest mirror + # dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_main=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No mirror URL found for ubuntu archive packages at ${source_url}." + exit 1 + fi + + # + # The internal mirror hosts the primary ubuntu package repository in the + # "ubuntu" subdirectory. + # + pkg_mirror_main+="ubuntu" fi +set -o errexit + +lb config \ + --parent-mirror-bootstrap "$pkg_mirror_main" \ + --parent-mirror-chroot "$pkg_mirror_main" \ + --parent-mirror-chroot-security "$pkg_mirror_main" \ + --parent-mirror-chroot-volatile "$pkg_mirror_main" \ + --parent-mirror-chroot-backports "$pkg_mirror_main" \ + --parent-mirror-binary "$pkg_mirror_main" \ + --parent-mirror-binary-security "$pkg_mirror_main" \ + --parent-mirror-binary-volatile "$pkg_mirror_main" \ + --parent-mirror-binary-backports "$pkg_mirror_main" \ + --mirror-bootstrap "$pkg_mirror_main" \ + --mirror-chroot "$pkg_mirror_main" \ + --mirror-chroot-security "$pkg_mirror_main" \ + --mirror-chroot-volatile "$pkg_mirror_main" \ + --mirror-chroot-backports "$pkg_mirror_main" \ + --mirror-binary "$pkg_mirror_main" \ + --mirror-binary-security "$pkg_mirror_main" \ + --mirror-binary-volatile "$pkg_mirror_main" \ + --mirror-binary-backports "$pkg_mirror_main" lb build From 631214be307b8c8c0205ff3c764f8ce1bede0cba Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 10 Oct 2019 14:47:56 -0700 Subject: [PATCH 004/197] [Backport to 6.0] Ensure that we are using secondary mirror (#372) --- live-build/auto/config | 18 ------------------ live-build/config/archives/ansible.key | Bin 1132 -> 0 bytes live-build/config/archives/ansible.list | 17 ----------------- .../archives/delphix-secondary-mirror.key | Bin 0 -> 2255 bytes .../archives/delphix-secondary-mirror.list.in | 3 ++- .../tasks/main.yml | 13 +------------ 6 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 live-build/config/archives/ansible.key delete mode 100644 live-build/config/archives/ansible.list create mode 100644 live-build/config/archives/delphix-secondary-mirror.key diff --git a/live-build/auto/config b/live-build/auto/config index e5c78fe1..2dc7b897 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -31,24 +31,6 @@ # delphix-platform. # lb config noauto \ - --parent-mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ --archive-areas "main restricted universe multiverse" \ --bootstrap-flavour minimal \ --chroot-filesystem none \ diff --git a/live-build/config/archives/ansible.key b/live-build/config/archives/ansible.key deleted file mode 100644 index 36ebbebd3a7b1c5e557b64eb1c2956ab2e8db650..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1132 zcmV-y1e5!j0u2OHmu)Zs5CF<8#y2sBavX?)67L*C?G(=zXq7e@f6Q6auif(;)LZod zJu=nD{xl;si4m1N>|mHxbs=?y#0s%cDFcoXWGuiOPPcBqgGtD(O&)chTnV3O_vTKp zpxjBtjm2l2l)Co$f!&#ZUZx+mr|if+XiX)imRGOaN{wFzF5qf*MMMWR-!r_9)GT8n z9-skmj@@r=PeO_JMTAEoMON^=mLX&{p9PrVXI@k~>Eol0;1Wto(EAo0j!ob9JLZv$ zz{J9ga|)i0iWfLtD7Lfx2pcwfa@C_G1MjY`Lz6(awYWneCUowKp!;(c|LRENDJuV@}J^R10+K>bBPj0 zCcgwPd7G4u#uSnVISiLz>kj2k1P55=&JfyHmd(KqdSL^A$;r(RK;Xi~)S07(S1`Hx zluz`k@CoV<@SfH`dGGH;tTOf>%sFH)iWJCW^83^D=M##Vb)#E2kuO`HOHZMJPPY`) z#rAzJVN>~#YhaCw6qRIee66;(wewmmFwLqfI#xM7Uyj4+Unp~Z9?dnFy^Ep&k9z1@*%FAsQAQg_q^ zZA|2Ax}@E=%BDtciOPD+c(7l!ho;1z9^v?+=mW{bPK7@|z`h3!+#5lH>mC;e zAZw{ca%yxWx*r-8+j=+B_0_#@aD^lrQGYcRci@}xrY_B1bATdXsNhK*{R`_5@Snvl z%@^X(#j_I0E(`LpP@0t-%adrgYf0nv{mkf^m#uEZhfVP0mH+ABM$$9YYGhc7uU_zb zLfwi01ixZuFJvRqgLiooRMNo7-bQbSnxUT8je#Ug{poZ{T)y9ck!Lmb=J$T22%i1(LzY!TWhePHBrdo)WGPY3)h z7C#hPORz)L!-36Ks;a^piJc|8(PTGnSad{%|NB2-0#?2=Ng~I(^p!6?WEWF|1MY|_ y zxKPcMYr<3pJMcg>$sX`efXd=13k#}T$VNIG=|Cy$?)&!gkYJ?_#3Pv#Ou;QI0gE!f z6uHR|e9TS1x6F=_^`V02gEP~%%Sl>oEtbaj2keKP*E6O}*OKv_hq$`g7UbAvSsKzZ9HkhdR-+tD8oJCZS~>gP;m@ z8~b1cNF0v@1)cza;cBs>Uk_OK)m2))n1!;zqC&#osjO9{WqlNG9;4_cF?mX@2!o zdQHE9yRU-ONb3L*0RRECBSK+wX>1@va&LDaJYr#UX>2ZIa&LD)WMyn{XlZyZV{dIf zi2_ap69EVSI2IuUnTFJG+#RXVPojPJv2@%0Ve))a1p->f=HCJv0|g5S2nPZN6$%Lm z3jzcd0s{d89svRufB*^!5V3UI{bBNaQ`Hd<{x?M!wg%G-9x%`ycOCrm)E@pP+}uI4 z<@^Ff;%=FKok6-NBFOUq=dT7|T^y++#Y(yVxH5Ao1_U9sEzIvw5(6ODb|oceZvwS0vvQY zI4iOqhbtbr7*5P+CI5nSG9IJ;GK8h{Csi9nkXF8N=y;U%Z(l+{dj_Q6y8E!^-g(YxJaGDYheD3%sL71vFP6c?rB83Q(X^XdjFB)^4(K0 z;DwjCtk$@e?zeHDmJEDe7CE%|;nH%9Mh-Y4&#>l(9gbX-idRhU8F>4mpsH06r5 z0ZWN8_p$?caNW*lxP9z9NpCzdUYJl3Tm{AAqb-AB<{jGI-1V~OLDJq7hGi&C@$F%p zIB~*E+Bz;Xo$?E~mG%y=7ir+Dleq#71X{=D-vJN+#nBb6;v%DPOJ`Bfem|)d)lry4 zd;D>G;<1`N7PF)jMU82Pk*N7Gn%=b+kHJye!ueFB5sO?BQH({o4^x-%>sdA_d9Ijc-5b3VsfFUv($?B)oz>m=kw|GDYJL~R30H%lA4&Epu_lb4*i>|E<|#-p95 zjV)Qz9*{S$wa2C#{{W-ho$~jT^W%xM!Y&UUJ`96@Jzm+p*JjVA-dRbqeAY>}*tu)D zCMi!N2!nw#RrAMxM^SL(3S)&VRmcU(=P4Ato*=*&9vHu6pkgaU z=7^1h1+t;p{P@tx=z%qAA3mj7&Ib?yrRjdd?gf^(dscqotumFD0dtF3l zQZ-yncp84%$blbM%wz8sPW1E!qr9E>E+lcPoG*v&tU!#<$Nk*bV3i@NOwGe7fJ~ql zsW0U_el`LO`0oyk_n91VHEoad23~EO01*KI0f_=O1Q-Db03a421eu1^aNHfK&`+X$ z__1``{bBNaQw0KA$L8Mx8w>yn2@tV#+x=nkd{c$a5B@OOK!M*BP5lWv=52whckhVec#b&Gp;6t8_hjPmex>PO&`WI_6LdkW_}b3W&!1lH|jyY@Ai;_3$f zwlcmd1R`TrR%;Hfit$T-t0qhxlK1eSl^Ke}@OYClx@r$%g|n518eFZD9VAIeyDJ%I za-y?Vu5o@j;|q%zTFO!&78C)k`e^JIbu|E=H|I&c?DstO*!mLlLoq)g-lE`BzQ?C% zsNzcb^BJhn;XRI#4f``ACbgMmn})~^ z;ddjYIk(4>=ML)t@hbV22E`r60rF03mf`eBVS0z|M~(i2}Zc zl5yXjXHr#KZcLMEcr4-Df(+-`TPh! dPuwM`LKr8?N literal 0 HcmV?d00001 diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 35816d28..3d42434f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,4 +14,5 @@ # limitations under the License. # -deb [trusted=yes] @@URL@@ bionic main +deb @@URL@@ bionic main multiverse universe +deb @@URL@@ bionic-updates main multiverse universe diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index 53894995..fcb8a962 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -51,23 +51,12 @@ # The virtualization package uses the /etc/issue file to store a # customer-supplied banner that is displayed prior to login. By default, # there should be no banner. Thus, we need to remove this file to -# acheive our desired behavior; Ubuntu ships with this file by default. +# achieve our desired behavior; Ubuntu ships with this file by default. # - file: path: /etc/issue state: absent -- apt_repository: - repo: "{{ item }}" - with_items: - - deb http://ddebs.ubuntu.com bionic main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-updates main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-proposed main restricted universe multiverse - retries: 3 - delay: 30 - register: result - until: result is succeeded - - shell: ls /lib/modules register: kernel_versions From d8c1629fff25e09d5ec26b1ca6dd43bfba678bc5 Mon Sep 17 00:00:00 2001 From: grodr <39814162+grodr@users.noreply.github.com> Date: Fri, 11 Oct 2019 10:55:44 -0700 Subject: [PATCH 005/197] [Backport of LB-209 to 6.0/patch] Want to consume new mirror for production appliance build (#376) --- branch.config | 13 +++++ scripts/build-ancillary-repository.sh | 13 +++-- scripts/docker-run.sh | 24 ++++++++ scripts/run-live-build.sh | 83 +++++++++++++++++++-------- 4 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 branch.config diff --git a/branch.config b/branch.config new file mode 100644 index 00000000..4a3b6bf9 --- /dev/null +++ b/branch.config @@ -0,0 +1,13 @@ +# +# Copyright (c) 2019 by Delphix. All rights reserved. +# + +# +# The "BRANCH" parameter tracks the upstream branch of appliance-build. It is +# used to determine which branch of the linux package mirror will be used for +# the build if UPSTREAM_PRODUCT_BRANCH is not set. UPSTREAM_PRODUCT_BRANCH is +# set when appliance build is built by the appliance-build Jenkins job. The +# UPSTREAM_BRANCH parameter should be updated by the release lead on branching +# + +UPSTREAM_BRANCH="6.0/patch" diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 047f1d74..133871fc 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -142,27 +142,30 @@ function build_ancillary_repository() { # environment variables, and the script will work as expected. # -upstream_branch="${UPSTREAM_PRODUCT_BRANCH:-master}" +if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH is not set." + exit 1 +fi AWS_S3_URI_VIRTUALIZATION=$(resolve_s3_uri \ "$AWS_S3_URI_VIRTUALIZATION" \ "$AWS_S3_PREFIX_VIRTUALIZATION" \ - "dlpx-app-gate/${upstream_branch}/build-package/post-push/latest") + "dlpx-app-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_MASKING=$(resolve_s3_uri \ "$AWS_S3_URI_MASKING" \ "$AWS_S3_PREFIX_MASKING" \ - "dms-core-gate/${upstream_branch}/build-package/post-push/latest") + "dms-core-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") AWS_S3_URI_USERLAND_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_USERLAND_PKGS" \ "$AWS_S3_PREFIX_USERLAND_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/userland/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/userland/post-push/latest") AWS_S3_URI_KERNEL_PKGS=$(resolve_s3_uri \ "$AWS_S3_URI_KERNEL_PKGS" \ "$AWS_S3_PREFIX_KERNEL_PKGS" \ - "devops-gate/master/linux-pkg-build/${upstream_branch}/kernel/post-push/latest") + "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/kernel/post-push/latest") # # All package files will be placed into this temporary directory, such diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index a4a240a0..733200c4 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -38,6 +38,29 @@ else DOCKER_RUN="docker run" fi +# +# Set UPSTREAM_BRANCH. This will determine which version of the linux package +# mirror is used. +# +if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + echo "UPSTREAM_PRODUCT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ + "Ensure branch.config is properly formatted with e.g." \ + "UPSTREAM_BRANCH=\"\"" + exit 1 + fi + echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." +else + UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" +fi +echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + $DOCKER_RUN --rm \ --privileged \ --network host \ @@ -64,6 +87,7 @@ $DOCKER_RUN --rm \ --env DELPHIX_SIGNATURE_VERSIONS \ --env DELPHIX_UPGRADE_MINIMUM_VERSION \ --env DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION \ + --env UPSTREAM_BRANCH="$UPSTREAM_BRANCH" \ --volume "$TOP:/opt/appliance-build" \ --workdir "/opt/appliance-build" \ appliance-build "$@" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 5783877a..4a01537b 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -127,37 +127,72 @@ while ! curl --output /dev/null --silent --head --fail \ sleep 1 done -set -o errexit +pkg_mirror_secondary='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_SECONDARY" ]]; then - sed "s|@@URL@@|$DELPHIX_PACKAGE_MIRROR_SECONDARY|" \ - config/archives/delphix-secondary-mirror.list + pkg_mirror_secondary="$DELPHIX_PACKAGE_MIRROR_SECONDARY" +else + # + # If no secondary package mirror is provided, then pull in the latest + # mirror dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_secondary=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No URL found for PPA packages at ${source_url}." + exit 1 + fi + + # The mirror hosts secondary packages in the "ppas" subdirectory. + pkg_mirror_secondary+="ppas" fi +sed "s|@@URL@@|$pkg_mirror_secondary|" \ + config/archives/delphix-secondary-mirror.list + +pkg_mirror_main='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then - lb config \ - --parent-mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --parent-mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-bootstrap "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-chroot-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-security "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-volatile "$DELPHIX_PACKAGE_MIRROR_MAIN" \ - --mirror-binary-backports "$DELPHIX_PACKAGE_MIRROR_MAIN" + pkg_mirror_main="$DELPHIX_PACKAGE_MIRROR_MAIN" else - lb config + # + # If no main package mirror is provided, then pull in the latest mirror + # dataset for the build. If no latest dataset is found, then fail. + # + source_url="http://linux-package-mirror.delphix.com/$UPSTREAM_BRANCH/latest/" + if ! pkg_mirror_main=$(curl -fLSs -o /dev/null -w '%{url_effective}' "$source_url"); then + kill -9 $APTLY_SERVE_PID + echo "No mirror URL found for ubuntu archive packages at ${source_url}." + exit 1 + fi + + # + # The internal mirror hosts the primary ubuntu package repository in the + # "ubuntu" subdirectory. + # + pkg_mirror_main+="ubuntu" fi +set -o errexit + +lb config \ + --parent-mirror-bootstrap "$pkg_mirror_main" \ + --parent-mirror-chroot "$pkg_mirror_main" \ + --parent-mirror-chroot-security "$pkg_mirror_main" \ + --parent-mirror-chroot-volatile "$pkg_mirror_main" \ + --parent-mirror-chroot-backports "$pkg_mirror_main" \ + --parent-mirror-binary "$pkg_mirror_main" \ + --parent-mirror-binary-security "$pkg_mirror_main" \ + --parent-mirror-binary-volatile "$pkg_mirror_main" \ + --parent-mirror-binary-backports "$pkg_mirror_main" \ + --mirror-bootstrap "$pkg_mirror_main" \ + --mirror-chroot "$pkg_mirror_main" \ + --mirror-chroot-security "$pkg_mirror_main" \ + --mirror-chroot-volatile "$pkg_mirror_main" \ + --mirror-chroot-backports "$pkg_mirror_main" \ + --mirror-binary "$pkg_mirror_main" \ + --mirror-binary-security "$pkg_mirror_main" \ + --mirror-binary-volatile "$pkg_mirror_main" \ + --mirror-binary-backports "$pkg_mirror_main" lb build From 05fa860f2972c44791821a3e403809cfea4ccd9b Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 11 Oct 2019 11:03:00 -0700 Subject: [PATCH 006/197] [Backport to 6.0] Ensure that we are using secondary mirror (#377) --- live-build/auto/config | 18 ------------------ live-build/config/archives/ansible.key | Bin 1132 -> 0 bytes live-build/config/archives/ansible.list | 17 ----------------- .../archives/delphix-secondary-mirror.key | Bin 0 -> 2255 bytes .../archives/delphix-secondary-mirror.list.in | 3 ++- .../tasks/main.yml | 13 +------------ 6 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 live-build/config/archives/ansible.key delete mode 100644 live-build/config/archives/ansible.list create mode 100644 live-build/config/archives/delphix-secondary-mirror.key diff --git a/live-build/auto/config b/live-build/auto/config index e5c78fe1..2dc7b897 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -31,24 +31,6 @@ # delphix-platform. # lb config noauto \ - --parent-mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --parent-mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-bootstrap "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-chroot-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-security "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-volatile "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ - --mirror-binary-backports "http://us-west-2.ec2.archive.ubuntu.com/ubuntu" \ --archive-areas "main restricted universe multiverse" \ --bootstrap-flavour minimal \ --chroot-filesystem none \ diff --git a/live-build/config/archives/ansible.key b/live-build/config/archives/ansible.key deleted file mode 100644 index 36ebbebd3a7b1c5e557b64eb1c2956ab2e8db650..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1132 zcmV-y1e5!j0u2OHmu)Zs5CF<8#y2sBavX?)67L*C?G(=zXq7e@f6Q6auif(;)LZod zJu=nD{xl;si4m1N>|mHxbs=?y#0s%cDFcoXWGuiOPPcBqgGtD(O&)chTnV3O_vTKp zpxjBtjm2l2l)Co$f!&#ZUZx+mr|if+XiX)imRGOaN{wFzF5qf*MMMWR-!r_9)GT8n z9-skmj@@r=PeO_JMTAEoMON^=mLX&{p9PrVXI@k~>Eol0;1Wto(EAo0j!ob9JLZv$ zz{J9ga|)i0iWfLtD7Lfx2pcwfa@C_G1MjY`Lz6(awYWneCUowKp!;(c|LRENDJuV@}J^R10+K>bBPj0 zCcgwPd7G4u#uSnVISiLz>kj2k1P55=&JfyHmd(KqdSL^A$;r(RK;Xi~)S07(S1`Hx zluz`k@CoV<@SfH`dGGH;tTOf>%sFH)iWJCW^83^D=M##Vb)#E2kuO`HOHZMJPPY`) z#rAzJVN>~#YhaCw6qRIee66;(wewmmFwLqfI#xM7Uyj4+Unp~Z9?dnFy^Ep&k9z1@*%FAsQAQg_q^ zZA|2Ax}@E=%BDtciOPD+c(7l!ho;1z9^v?+=mW{bPK7@|z`h3!+#5lH>mC;e zAZw{ca%yxWx*r-8+j=+B_0_#@aD^lrQGYcRci@}xrY_B1bATdXsNhK*{R`_5@Snvl z%@^X(#j_I0E(`LpP@0t-%adrgYf0nv{mkf^m#uEZhfVP0mH+ABM$$9YYGhc7uU_zb zLfwi01ixZuFJvRqgLiooRMNo7-bQbSnxUT8je#Ug{poZ{T)y9ck!Lmb=J$T22%i1(LzY!TWhePHBrdo)WGPY3)h z7C#hPORz)L!-36Ks;a^piJc|8(PTGnSad{%|NB2-0#?2=Ng~I(^p!6?WEWF|1MY|_ y zxKPcMYr<3pJMcg>$sX`efXd=13k#}T$VNIG=|Cy$?)&!gkYJ?_#3Pv#Ou;QI0gE!f z6uHR|e9TS1x6F=_^`V02gEP~%%Sl>oEtbaj2keKP*E6O}*OKv_hq$`g7UbAvSsKzZ9HkhdR-+tD8oJCZS~>gP;m@ z8~b1cNF0v@1)cza;cBs>Uk_OK)m2))n1!;zqC&#osjO9{WqlNG9;4_cF?mX@2!o zdQHE9yRU-ONb3L*0RRECBSK+wX>1@va&LDaJYr#UX>2ZIa&LD)WMyn{XlZyZV{dIf zi2_ap69EVSI2IuUnTFJG+#RXVPojPJv2@%0Ve))a1p->f=HCJv0|g5S2nPZN6$%Lm z3jzcd0s{d89svRufB*^!5V3UI{bBNaQ`Hd<{x?M!wg%G-9x%`ycOCrm)E@pP+}uI4 z<@^Ff;%=FKok6-NBFOUq=dT7|T^y++#Y(yVxH5Ao1_U9sEzIvw5(6ODb|oceZvwS0vvQY zI4iOqhbtbr7*5P+CI5nSG9IJ;GK8h{Csi9nkXF8N=y;U%Z(l+{dj_Q6y8E!^-g(YxJaGDYheD3%sL71vFP6c?rB83Q(X^XdjFB)^4(K0 z;DwjCtk$@e?zeHDmJEDe7CE%|;nH%9Mh-Y4&#>l(9gbX-idRhU8F>4mpsH06r5 z0ZWN8_p$?caNW*lxP9z9NpCzdUYJl3Tm{AAqb-AB<{jGI-1V~OLDJq7hGi&C@$F%p zIB~*E+Bz;Xo$?E~mG%y=7ir+Dleq#71X{=D-vJN+#nBb6;v%DPOJ`Bfem|)d)lry4 zd;D>G;<1`N7PF)jMU82Pk*N7Gn%=b+kHJye!ueFB5sO?BQH({o4^x-%>sdA_d9Ijc-5b3VsfFUv($?B)oz>m=kw|GDYJL~R30H%lA4&Epu_lb4*i>|E<|#-p95 zjV)Qz9*{S$wa2C#{{W-ho$~jT^W%xM!Y&UUJ`96@Jzm+p*JjVA-dRbqeAY>}*tu)D zCMi!N2!nw#RrAMxM^SL(3S)&VRmcU(=P4Ato*=*&9vHu6pkgaU z=7^1h1+t;p{P@tx=z%qAA3mj7&Ib?yrRjdd?gf^(dscqotumFD0dtF3l zQZ-yncp84%$blbM%wz8sPW1E!qr9E>E+lcPoG*v&tU!#<$Nk*bV3i@NOwGe7fJ~ql zsW0U_el`LO`0oyk_n91VHEoad23~EO01*KI0f_=O1Q-Db03a421eu1^aNHfK&`+X$ z__1``{bBNaQw0KA$L8Mx8w>yn2@tV#+x=nkd{c$a5B@OOK!M*BP5lWv=52whckhVec#b&Gp;6t8_hjPmex>PO&`WI_6LdkW_}b3W&!1lH|jyY@Ai;_3$f zwlcmd1R`TrR%;Hfit$T-t0qhxlK1eSl^Ke}@OYClx@r$%g|n518eFZD9VAIeyDJ%I za-y?Vu5o@j;|q%zTFO!&78C)k`e^JIbu|E=H|I&c?DstO*!mLlLoq)g-lE`BzQ?C% zsNzcb^BJhn;XRI#4f``ACbgMmn})~^ z;ddjYIk(4>=ML)t@hbV22E`r60rF03mf`eBVS0z|M~(i2}Zc zl5yXjXHr#KZcLMEcr4-Df(+-`TPh! dPuwM`LKr8?N literal 0 HcmV?d00001 diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 35816d28..3d42434f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,4 +14,5 @@ # limitations under the License. # -deb [trusted=yes] @@URL@@ bionic main +deb @@URL@@ bionic main multiverse universe +deb @@URL@@ bionic-updates main multiverse universe diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index 53894995..fcb8a962 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -51,23 +51,12 @@ # The virtualization package uses the /etc/issue file to store a # customer-supplied banner that is displayed prior to login. By default, # there should be no banner. Thus, we need to remove this file to -# acheive our desired behavior; Ubuntu ships with this file by default. +# achieve our desired behavior; Ubuntu ships with this file by default. # - file: path: /etc/issue state: absent -- apt_repository: - repo: "{{ item }}" - with_items: - - deb http://ddebs.ubuntu.com bionic main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-updates main restricted universe multiverse - - deb http://ddebs.ubuntu.com bionic-proposed main restricted universe multiverse - retries: 3 - delay: 30 - register: result - until: result is succeeded - - shell: ls /lib/modules register: kernel_versions From 983910b2424ba02e16d2f889dadb8685bc7f2a1d Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 14 Oct 2019 10:17:26 -0700 Subject: [PATCH 007/197] DLPX-66713 [Backport of Issue DLPX-66369 to 6.0.1.0] Using "upgrade-container stop" may not stop the running upgrade container (#375) --- upgrade/upgrade-scripts/upgrade-container | 26 +++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 7c69ded8..6e9314f0 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -376,11 +376,11 @@ function start() { die "container '$CONTAINER' non-existent or mis-configured" fi - systemctl start "systemd-nspawn@$CONTAINER" || + machinectl start "$CONTAINER" || die "failed to start container '$CONTAINER'" # - # When starting the container above with "systemctl start", the + # When starting the container above with "machinectl start", the # command will return prior to the container having fully # completed it's boot process. Thus, to ensure we don't attempt # the verification steps prior to the container being ready, we @@ -400,8 +400,26 @@ function start() { } function stop() { - systemctl stop "systemd-nspawn@$CONTAINER" || - die "failed to stop container: '$CONTAINER'" + machinectl terminate "$CONTAINER" || + die "failed to termiante container: '$CONTAINER'" + + # + # The above command used to terminate the container is not + # synchronous, and may return before termination of the + # container is complete. Thus, we poll here, and wait for the + # the termination to complete before continuing. + # + # shellcheck disable=SC2034 + for i in {1..600}; do + if ! machinectl status "$CONTAINER" &>/dev/null; then + break + fi + + sleep 1 + done + + machinectl status "$CONTAINER" &>/dev/null && + die "timeout waiting for container termination: '$CONTAINER'" } function destroy() { From acf8cb2e16de7efe0ac80f5faf3766e0f6c5b64c Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 16 Oct 2019 13:04:34 -0700 Subject: [PATCH 008/197] Make debug symbol packages (ddebs) available for installation (#378) --- scripts/build-ancillary-repository.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 133871fc..af55214d 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -86,7 +86,7 @@ function download_delphix_s3_debs() { aws s3 sync --only-show-errors "$S3_URI" . sha256sum -c --strict SHA256SUMS - mv ./*.deb "$pkg_directory/" + mv ./*deb "$pkg_directory/" popd &>/dev/null rm -rf "$tmp_directory" From 64753c01ab473700c19b7adb1448bc358786feeb Mon Sep 17 00:00:00 2001 From: Sara Hartse Date: Wed, 30 Oct 2019 10:37:51 -0700 Subject: [PATCH 009/197] DLPX-66331 [Backport of Issue DLPX-66286 to 6.0.0.0] Interrupting a running upgrade verify can leave the engine not-upgradable (#385) --- live-build/misc/migration-scripts/dx_apply | 21 ++---- live-build/misc/migration-scripts/dx_delete | 82 ++++++++++++++++++++- live-build/misc/migration-scripts/dx_verify | 4 +- 3 files changed, 85 insertions(+), 22 deletions(-) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 05e468b6..23121740 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -113,21 +113,12 @@ rm -f /boot/vmlinuz-* /boot/initrd.img-* || zfs destroy -r "$RPOOL/ROOT" 2>/dev/null zfs list "$RPOOL/ROOT" 2>/dev/null && die "could not destroy linux root dataset from previous run" -sed -i '/set lxcmdline/d' /boot/menu.rc.local -[[ "$(grep -cF 'set lxcmdline' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup lxcmdline from previous run" -sed -i '/set mainmenu_caption\[8\]/d' /boot/menu.rc.local -[[ "$(grep -cF 'mainmenu_caption[8]' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup mainmenu_caption from previous run" -sed -i '/set mainansi_caption\[8\]/d' /boot/menu.rc.local -[[ "$(grep -cF 'mainansi_caption[8]' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup mainansi_caption from previous run" -sed -i '/set mainmenu_keycode\[8\]/d' /boot/menu.rc.local -[[ "$(grep -cF 'mainmenu_keycode[8]' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup mainmenu_keycode from previous run" -sed -i '/set mainmenu_command\[8\]/d' /boot/menu.rc.local -[[ "$(grep -cF 'mainmenu_command[8]' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup mainmenu_command from previous run" + +# +# Save a copy of the boot menu to restore if we aren't upgrading +# +cp "/boot/menu.rc.local" "/boot/menu.rc.local.copy" || + die "failed to save copy of /boot/menu.rc.local" report_progress_inc 20 diff --git a/live-build/misc/migration-scripts/dx_delete b/live-build/misc/migration-scripts/dx_delete index 08fc03af..f90a764e 100755 --- a/live-build/misc/migration-scripts/dx_delete +++ b/live-build/misc/migration-scripts/dx_delete @@ -39,10 +39,84 @@ function die() { # names are randomly generated. # -linux_dataset="rpool/ROOT" -if ! zfs list ${linux_dataset} &>/dev/null; then - echo "Linux dataset '${linux_dataset}' is not installed." +LX_DATASET="rpool/ROOT" +if ! zfs list ${LX_DATASET} &>/dev/null; then + echo "Linux dataset '${LX_DATASET}' is not installed." exit 0 fi -zfs destroy -r ${linux_dataset} || die "Failed to destroy Linux dataset '${linux_dataset}'" +LX_RDS_PARENT=$(zfs list -o name -H -d 1 "$LX_DATASET" | tail -n 1) +[[ -n $LX_RDS_PARENT ]] || die "could not find Linux RDS parent dataset" +LX_CONTAINER="${LX_RDS_PARENT##*/}" +LX_RDS_TMP_ROOT_MOUNT="/tmp/$LX_CONTAINER/root" + +MDS_SNAPNAME="MDS-CLONE-upgradeverify" +MDS_CLONE=domain0/$MDS_SNAPNAME +SVC=svc:/system/delphix/postgres:$MDS_SNAPNAME +PG_DATA=/$MDS_CLONE/db + +BOOT_MENU=/boot/menu.rc.local +BOOT_MENU_COPY=$BOOT_MENU.copy + +function dx_apply_cleanup() { + # + # Restore original version of the bootloader + # + if [[ -e $BOOT_MENU_COPY ]]; then + mv $BOOT_MENU_COPY $BOOT_MENU || + die "failed to restore copy of $BOOT_MENU" + fi + + rm -f /boot/vmlinuz-* /boot/initrd.img-* || + die "failed to destroy previously copied Linux kernel data" +} + +function dx_verify_cleanup() { + # + # dx_verify creates a clone of the MDS dataset and the postgres + # service as well as a clone of /var/delphix for masking validation. + # If any are left, clean them up. (This logic is based on dx_manage_pg + # stop and cleanup.) + # + local sta + sta=$(svcs -Ho sta $SVC) + if [[ -n "$sta" ]]; then + /usr/sbin/svcadm disable -s $SVC || + die "unable to disable SMF service: $SVC" + /usr/sbin/svccfg delete -f $SVC || + die "unable to delete SMF service: $SVC" + fi + + # + # Clean up the postmaster.pid leftover from the main MDS in the + # snapshot. This needs to be done after disabling the SMF service in + # case the postmaster.pid was instead from the active clone. + # + if [[ -e $PG_DATA/postmaster.pid ]]; then + echo "renaming $PG_DATA/postmaster.pid to $PG_DATA/postmaster.pid.original" + mv $PG_DATA/postmaster.pid $PG_DATA/postmaster.pid.original || + echo "failed to move postmaster.pid file" + fi + + [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && + zfs destroy -R "domain0/mds@$MDS_SNAPNAME" + [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && + die "unable to cleanup domain0/mds@$MDS_SNAPNAME" + + runningVar=$(mount | awk '/^\/var\/delphix /{ print $3 }') + [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && + zfs destroy -R "$runningVar@$MDS_SNAPNAME" + [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && + die "unable to cleanup $runningVar@$MDS_SNAPNAME" +} + +dx_apply_cleanup +dx_verify_cleanup + +# +# Recursively destroy and force-unmount the Linux Root Dataset. Even though all the +# datasets are cleaned up in a normal execution of Verify, this handles the case when +# it was cancelled early. +# +zfs destroy -rf ${LX_DATASET} || die "Failed to destroy Linux dataset '${LX_DATASET}'" +rm -rf "$LX_RDS_TMP_ROOT_MOUNT" diff --git a/live-build/misc/migration-scripts/dx_verify b/live-build/misc/migration-scripts/dx_verify index 0e271a62..86a5ccbd 100755 --- a/live-build/misc/migration-scripts/dx_verify +++ b/live-build/misc/migration-scripts/dx_verify @@ -104,15 +104,13 @@ function mount_datasets() { die "unable to create snapshot $runningVar@$MDS_SNAPNAME" zfs clone "$runningVar@$MDS_SNAPNAME" "$runningVar/$MDS_SNAPNAME" || die "unable to create $runningVar/$MDS_SNAPNAME" - zfs destroy -d "$runningVar@$MDS_SNAPNAME" || - die "unable to defer destroy $runningVar@$MDS_SNAPNAME" mount -F zfs "$runningVar/$MDS_SNAPNAME" "$root/var/delphix" || die "unable to mount $root/var/delphix" } function cleanup_datasets() { if [[ -n "$runningVar" ]]; then - zfs destroy -f "$runningVar/$MDS_SNAPNAME" + zfs destroy -R "$runningVar@$MDS_SNAPNAME" fi umount -f "$root" rmdir "$root" From 3eba828e7b0a442a7abd913c59ddb83276d853f6 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Wed, 30 Oct 2019 20:08:21 -0600 Subject: [PATCH 010/197] [Backport to 6.0.0.0] Set 'elevator=noop' GRUB option for first boot after migration (#386) --- live-build/misc/migration-scripts/dx_apply | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 23121740..2fa2f1f2 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -274,6 +274,7 @@ LX_CMDLINE=( 'crashkernel=256M,low' 'zfsforce=1' 'mitigations=off' + 'elevator=noop' ) # From b5ef99257f689d316d3efac57837627e6491daa2 Mon Sep 17 00:00:00 2001 From: Sara Hartse Date: Tue, 12 Nov 2019 07:21:56 -0800 Subject: [PATCH 011/197] DLPX-66697 [Backport of Issue DLPX-66696 to 6.0.0.0] migration: add stress options in the pre-reboot dx scripts (#390) --- live-build/misc/migration-scripts/dx_apply | 7 +++++++ live-build/misc/migration-scripts/dx_execute | 10 +++++++++- .../misc/migration-scripts/dx_upg_stress_options | 14 ++++++++++++-- live-build/misc/migration-scripts/dx_verify | 7 ++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 2fa2f1f2..79df8cb7 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -97,6 +97,11 @@ MIGRATION_MINOR_VERSION=$(echo "$MIN_MIGRATION_VERSION" | cut -d. -f3) die "expected version $MIN_MIGRATION_VERSION or greater but found" \ "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" +DX_UPG_STRESS=$ARCHIVE_DIR/dx_upg_stress_options +# shellcheck source=/dev/null +. $DX_UPG_STRESS --source +__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_AFTER_VERSION_CHECK" + # # Get the root dataset and the current ZFS pool that we're currently using. # @@ -338,6 +343,8 @@ MAIN_MENU_FICL=( die "linux dataset for /var/delphix contains unexpected files" ) || die "verification of /var/delphix failed" +__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING" + umount "$TMP_ROOT/var/log" || die "couldn't unmount linux dataset $TMP_ROOT/var/log" zfs destroy "$RPOOL/ROOT/$FSNAME/data" || diff --git a/live-build/misc/migration-scripts/dx_execute b/live-build/misc/migration-scripts/dx_execute index ee005fa3..fdef0ca6 100755 --- a/live-build/misc/migration-scripts/dx_execute +++ b/live-build/misc/migration-scripts/dx_execute @@ -33,6 +33,9 @@ umask 0022 set -o pipefail DX_UPG_PAUSE="${BASH_SOURCE%/*}/dx_upg_pause_options" +DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" +# shellcheck source=/dev/null +. $DX_UPG_STRESS --source function die() { echo "$(basename "$0"): $*" >&2 @@ -58,7 +61,7 @@ function cleanup_leftover_dataset() { local dataset=$1 if zfs list "$dataset" &>/dev/null; then - zfs destroy "$dataset" || + zfs destroy -R "$dataset" || die "Failed to destroy leftover dataset '$dataset'" fi } @@ -129,6 +132,9 @@ mount -F zfs -o ignoremountpoint "$LX_RDS" "$LX_RDS_MOUNT" || die "failed to mount the root Linux dataset" "${LX_RDS_MOUNT}${MIGRATION_SCRIPT}" pre-upgrade \ >>/var/delphix/migration/log 2>&1 || die "failed to run migration" + +__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION" + # # Create a flag file that notifies the delphix-migration service that # post-reboot migration logic should be run. Note that the /var/delphix @@ -214,6 +220,8 @@ timeout 10 /opt/delphix/server/bin/jmxtool boot upgrade server $DX_UPG_PAUSE --pause "PAUSE_IN_DX_EXECUTE_BEFORE_RESTART" || die "failed to pause fully on stress option" +__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT" + # Constants used by the uadmin syscall. A_SHUTDOWN=2 AD_POWEROFF=6 diff --git a/live-build/misc/migration-scripts/dx_upg_stress_options b/live-build/misc/migration-scripts/dx_upg_stress_options index eb39b963..addd500f 100755 --- a/live-build/misc/migration-scripts/dx_upg_stress_options +++ b/live-build/misc/migration-scripts/dx_upg_stress_options @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2016, 2017 by Delphix. All rights reserved. +# Copyright (c) 2016, 2019 by Delphix. All rights reserved. # # @@ -22,6 +22,11 @@ __STRESS_OPTIONS_JSON=$( "err_msg": "Stress option triggered after version check.", "auto_unset": true }, + "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING": { + "location": "pre-reboot", + "err_msg": "Stress option triggered before dx_apply dataset cleanup.", + "auto_unset": true + }, "STRESS_DX_INSTALL_ARCHIVE_FAIL_AFTER_VERIFY_DX_ARCHIVE": { "location": "pre-reboot", "err_msg": "Stress option triggered after verify_dx_archive.", @@ -34,7 +39,12 @@ __STRESS_OPTIONS_JSON=$( }, "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION": { "location": "pre-reboot", - "err_msg": "Stress option triggered after testing migration.", + "err_msg": "Stress option triggered after upgrade-verify.jar.", + "auto_unset": true + }, + "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION": { + "location": "pre-reboot", + "err_msg": "Stress option triggered after migrating OS config.", "auto_unset": true }, "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT": { diff --git a/live-build/misc/migration-scripts/dx_verify b/live-build/misc/migration-scripts/dx_verify index 86a5ccbd..84b9368b 100755 --- a/live-build/misc/migration-scripts/dx_verify +++ b/live-build/misc/migration-scripts/dx_verify @@ -198,7 +198,12 @@ report_progress_inc 0 "preparing for verification" mount_datasets "$version" report_progress_inc 20 "running upgrade checks" run_upgrade_verify "$output" "$format" "$locale" 20 95 -#TODO LX-1808 stress options + +DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" +# shellcheck source=/dev/null +. $DX_UPG_STRESS --source +__trigger_unset_stress_option "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION" + test_masking report_progress_inc 95 "cleaning up post-verification" cleanup From 8ca15c1904f3375ed3105abbbe4f9138f5eb8c4a Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 12 Nov 2019 10:22:25 -0500 Subject: [PATCH 012/197] DLPX-67336 [Backport of Issue DLPX-67184 to 6.0.0.0] migration: dx_apply should call dx_delete to cleanup previous state (#389) --- live-build/misc/migration-scripts/dx_apply | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 79df8cb7..75efc62b 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -111,13 +111,8 @@ RPOOL=${RDS%%/*} # # Cleanup any previous intermediate state. # -rm -rf /tmp/delphix.* || - die "failed to destroy old delphix temporary directories" -rm -f /boot/vmlinuz-* /boot/initrd.img-* || - die "failed to destroy previously copied Linux kernel data" -zfs destroy -r "$RPOOL/ROOT" 2>/dev/null -zfs list "$RPOOL/ROOT" 2>/dev/null && - die "could not destroy linux root dataset from previous run" +"${BASH_SOURCE%/*}/dx_delete" || + die "failed to cleanup previous state with dx_delete" # # Save a copy of the boot menu to restore if we aren't upgrading From 4b82c5f4131657acce9f3bbe2bf0d5a05ed1be0a Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Tue, 12 Nov 2019 22:52:15 -0500 Subject: [PATCH 013/197] DLPX-67355 [Backport of Issue DLPX-66957 to 6.0.0.0] hostname is not locally resolvable (#392) At boot time, applications will fail to resolve the local hostname to an IP address (i.e. the loopback address) until DNS servers are reachable, and once they are reachable, such lookups will result in external DNS requests, which is suboptimal given how often applications resolve the local hostname. These lookups should all be done locally without relying on external DNS servers. When applications resolve a hostname to an IP address, they use the NSS switch, which has a policy for how lookups should be done. On Delphix, the hosts entry for /etc/nsswitch.conf looks like this: hosts: files dns mymachines This means that any lookup will first consult local files (/etc/hosts), and if that fails, will go out to DNS. The /etc/hosts file is managed by cloud-init, and will contain the local hostname in cases where Delphix is running in the cloud. However, when Delphix is not in a cloud environment (e.g. on ESX), cloud-init does not update /etc/hosts with the local hostname (e.g. when it comes from a DHCP option). Lookups then end up going to DNS. The DNS configuration is in /etc/resolv.conf. Today, this file is a symbolic link to /run/systemd/resolve/resolv.conf, which in a DHCP environment, contains the set of DNS servers obtained via DHCP. As such, even requests to resolve the local hostname will end up going out to a remote DNS server. Ubuntu does, however, ship with a local caching DNS server that is built into systemd-resolved, and this caching DNS server has built-in support for resolving the local hostname without having to send such queries externally. Delphix doesn't use this because that would require that /etc/resolv.conf include a "localhost" DNS server so that DNS queries are made to the local DNS server. This can be accomplished by having /etc/resolv.conf be a symbolic link to /run/systemd/resolve/stub-resolv.conf. This resolver configuration points to the localhost server (systemd-resolved), which forwards queries that it wasn't able to satisfy from its local cache to external servers. This also happens to be the recommended default configuration according to the systemd-resolved.service man page. This fix simply changes the default symlink target for /etc/resolv.conf to be /run/systemd/resolve/stub-resolv.conf. --- .../config/hooks/vm-artifacts/83-etc-resolv-conf.binary | 2 +- upgrade/upgrade-scripts/upgrade-container | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/83-etc-resolv-conf.binary b/live-build/config/hooks/vm-artifacts/83-etc-resolv-conf.binary index ea45e80c..5718ddbd 100755 --- a/live-build/config/hooks/vm-artifacts/83-etc-resolv-conf.binary +++ b/live-build/config/hooks/vm-artifacts/83-etc-resolv-conf.binary @@ -31,4 +31,4 @@ # environment. # chroot binary rm -f /etc/resolv.conf -chroot binary ln -s ../run/systemd/resolve/resolv.conf /etc/resolv.conf +chroot binary ln -s ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 7c69ded8..3aa368e5 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -589,15 +589,6 @@ function migrate_configuration() { # system to properly operate; these must be preserved acorss # upgrades. # - # Note, the "/etc/resolv.conf" file is in this list, but the - # "migrate_file" function will only migrate this path if it - # points to a regular file. Often, this file will be a symlink, - # in which case it won't be migrated, and we don't want it to - # be. We only want to migrate this file if the admin (or some - # software on the system) has converted it to a regular file, - # and configured some host specific DNS settings, in which case - # we want these settings to persist across the upgrade. - # while read -r file; do migrate_file "$file" done <<-EOF From 62ff62452d48fa6d5d872950f033e7c487ca4598 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Fri, 15 Nov 2019 14:53:41 -0500 Subject: [PATCH 014/197] DLPX-67393 [Backport of Issue DLPX-67392 to 6.0.0.0] azure marketplace requires a 1mb unallocated region (#394) --- .../config/hooks/vm-artifacts/90-raw-disk-image.binary | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 020d97c6..8d0e5623 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -64,10 +64,12 @@ sgdisk --zap-all "$ARTIFACT_NAME.img" # partition will be used and automatically detected by "grub-install" # based on the partitions typecode. This partition is required since we're # partitioning using GPT; if we used MBR, an explicit boot partition -# wouldn't be required. +# wouldn't be required. Also we leave the first 1MB unallocated since +# some clouds (i.e. Azure) may require space for their own internal +# purposes. # sgdisk "$ARTIFACT_NAME.img" \ - --set-alignment=1 --new=2:34:2047 --typecode=2:EF02 + --set-alignment=1 --new=2:1m:+1m --typecode=2:EF02 # # Now we create the partition that we'll use for the zpool that will be From 2efee20454bc0b172e1d900f3db3f8cc7bb7b201 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Tue, 7 Jan 2020 18:37:37 -0500 Subject: [PATCH 015/197] DLPX-67946 [Backport of Issue DLPX-67884 to 6.0.0.0] need Hyper-V image (#403) --- gradle-lib/util.gradle | 4 ++-- live-build/build.gradle | 3 ++- .../config/hooks/vm-artifacts/91-vhdx-disk-image.binary | 5 +++-- scripts/run-live-build.sh | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/gradle-lib/util.gradle b/gradle-lib/util.gradle index b7064329..abb1b00b 100644 --- a/gradle-lib/util.gradle +++ b/gradle-lib/util.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2019 Delphix + * Copyright 2019, 2020 Delphix * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import java.nio.file.Paths import java.nio.file.attribute.PosixFilePermissions import java.util.regex.Pattern -ext.allPlatforms = ["aws", "azure", "esx", "gcp", "kvm"] +ext.allPlatforms = ["aws", "azure", "esx", "gcp", "hyperv", "kvm"] ext.allVariants = new File("${rootProject.projectDir}/live-build/variants").list() ext.allInternal = allVariants.findAll { it.startsWith("internal-") } ext.allExternal = allVariants.findAll { it.startsWith("external-") } diff --git a/live-build/build.gradle b/live-build/build.gradle index eb73886d..24c2480c 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2019 Delphix + * Copyright 2019, 2020 Delphix * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,6 +46,7 @@ def artifactTypes = ["aws": "vmdk", "azure": "vhdx", "esx": "ova", "gcp": "gcp.tar.gz", + "hyperv": "vhdx", "kvm": "qcow2"] def configDir = "configuration" diff --git a/live-build/config/hooks/vm-artifacts/91-vhdx-disk-image.binary b/live-build/config/hooks/vm-artifacts/91-vhdx-disk-image.binary index fed843cd..6cc04c98 100755 --- a/live-build/config/hooks/vm-artifacts/91-vhdx-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/91-vhdx-disk-image.binary @@ -1,6 +1,6 @@ #!/bin/bash -ex # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ # assumed to be a "raw" disk image, into a VHDX formated disk image. # -[[ "$APPLIANCE_PLATFORM" == azure ]] || exit 0 +[[ "$APPLIANCE_PLATFORM" == azure ]] || + [[ "$APPLIANCE_PLATFORM" == hyperv ]] || exit 0 qemu-img convert "$ARTIFACT_NAME.img" \ -O vhdx -o subformat=dynamic "$ARTIFACT_NAME.vhdx" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 4a01537b..482ccce6 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -217,6 +217,7 @@ aws) vm_artifact_ext=vmdk ;; azure) vm_artifact_ext=vhdx ;; esx) vm_artifact_ext=ova ;; gcp) vm_artifact_ext=gcp.tar.gz ;; +hyperv) vm_artifact_ext=vhdx ;; kvm) vm_artifact_ext=qcow2 ;; *) echo "Invalid platform" From ed09d0980d5ff02705bc79fdf0d94bfb4e5f21f0 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Jan 2020 08:56:11 -0800 Subject: [PATCH 016/197] Stop using Docker when executing the build (#407) * Disable integration with Jenkins automation * Leverage Github Actions in favor of TravisCI (#398) This change adds the necessary files and logic to take advantage of GitHub Actions for running our automated tests. Currently we rely on TravisCI to do this, but we'd like to transition over to GitHub Actions, for the following reasons: 1. The Actions UI is more integrated with GitHub PRs 2. Best practices at Delphix suggests using Actions Additionally, this change removes integration with the "bors" bot. * Stop using Docker when executing the build (#404) --- .github/scripts/install-ansible-lint.sh | 15 ++ .github/scripts/install-gradle.sh | 6 + .github/scripts/install-shellcheck.sh | 4 + .github/scripts/install-shfmt.sh | 5 + .github/workflows/main.yml | 24 +++ .gitignore | 2 +- .travis.yml | 22 --- README.md | 57 ++---- .../appliance-build.bootstrap/tasks/main.yml | 38 +++- bors.toml | 16 -- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 55190 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 172 ++++++++++++++++++ scripts/build-ancillary-repository.sh | 24 ++- scripts/run-live-build.sh | 23 +++ 15 files changed, 327 insertions(+), 87 deletions(-) create mode 100755 .github/scripts/install-ansible-lint.sh create mode 100755 .github/scripts/install-gradle.sh create mode 100755 .github/scripts/install-shellcheck.sh create mode 100755 .github/scripts/install-shfmt.sh create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml delete mode 100644 bors.toml create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew diff --git a/.github/scripts/install-ansible-lint.sh b/.github/scripts/install-ansible-lint.sh new file mode 100755 index 00000000..53b73b39 --- /dev/null +++ b/.github/scripts/install-ansible-lint.sh @@ -0,0 +1,15 @@ +#!/bin/bash -ex + +git clone https://github.com/willthames/ansible-lint /opt/ansible-lint +cd /opt/ansible-lint +git checkout v3.4.21 +git branch -D master + +# +# GitHub Actions exposes some "debugging commands" that can be used to +# manipulate the environment of the job that's running. In this case, we +# use the "set-env" command to modify the environment of the job, to +# edit the PATH and PYTHONPATH global variables. +# +echo "::set-env name=PATH::${PATH}:/opt/ansible-lint/bin" +echo "::set-env name=PYTHONPATH::${PYTHONPATH}:/opt/ansible-lint/lib" diff --git a/.github/scripts/install-gradle.sh b/.github/scripts/install-gradle.sh new file mode 100755 index 00000000..95497da5 --- /dev/null +++ b/.github/scripts/install-gradle.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +wget -nv https://services.gradle.org/distributions/gradle-5.1-bin.zip +sha256sum -c <<< '7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d gradle-5.1-bin.zip' +unzip -d /opt gradle-5.1-bin.zip +rm gradle-5.1-bin.zip diff --git a/.github/scripts/install-shellcheck.sh b/.github/scripts/install-shellcheck.sh new file mode 100755 index 00000000..15348ec4 --- /dev/null +++ b/.github/scripts/install-shellcheck.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex + +apt-get update +apt-get install -y shellcheck diff --git a/.github/scripts/install-shfmt.sh b/.github/scripts/install-shfmt.sh new file mode 100755 index 00000000..f7567094 --- /dev/null +++ b/.github/scripts/install-shfmt.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex + +wget -nv -O /usr/local/bin/shfmt \ + https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_amd64 +chmod +x /usr/local/bin/shfmt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..c27a55f4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,24 @@ +on: [push, pull_request] + +jobs: + check-ansible: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: sudo ./.github/scripts/install-gradle.sh + - run: sudo ./.github/scripts/install-ansible-lint.sh + - run: /opt/gradle-5.1/bin/gradle ansibleCheck + check-shellcheck: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: sudo ./.github/scripts/install-gradle.sh + - run: sudo ./.github/scripts/install-shellcheck.sh + - run: /opt/gradle-5.1/bin/gradle shellCheck + check-shfmt: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: sudo ./.github/scripts/install-gradle.sh + - run: sudo ./.github/scripts/install-shfmt.sh + - run: /opt/gradle-5.1/bin/gradle shfmtCheck diff --git a/.gitignore b/.gitignore index 64329d3f..64caed74 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # -# Copyright 2018 Delphix +# Copyright 2018-2019 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 020ddc5e..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: generic -sudo: required - -branches: - only: - - staging - - trying - - master - -services: - - docker - -env: - - TARGET=ansibleCheck - - TARGET=shellCheck - - TARGET=shfmtCheck - -install: - - docker build -t appliance-build:latest docker - -script: - - ./scripts/docker-run.sh gradle $TARGET diff --git a/README.md b/README.md index 6baa8979..a22bdd99 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delphix Appliance Build -[![Build Status](https://travis-ci.com/delphix/appliance-build.svg?branch=master)](https://travis-ci.com/delphix/appliance-build) +![](https://github.com/delphix/appliance-build/workflows/.github/workflows/main.yml/badge.svg) This repository contains the code used to build the Ubuntu-based Delphix Appliance, leveraging open-source tools such as Debian's live-build, @@ -20,7 +20,7 @@ Log into that VM using the "ubuntu" user, and run these commands: $ git clone https://github.com/delphix/appliance-build.git $ cd appliance-build $ ansible-playbook bootstrap/playbook.yml - $ ./scripts/docker-run.sh gradle buildInternalMinimalKvm + $ sudo ./gradlew buildInternalMinimalKvm $ sudo qemu-system-x86_64 -nographic -m 1G \ > -drive file=live-build/build/artifacts/internal-minimal-kvm.qcow2 @@ -31,29 +31,18 @@ To exit "qemu", use "Ctrl-A X". The Delphix Appliance build system has the following assuptions about the environment from which it will be executed: - 1. Docker must be installed and available to be used on the host - that'll run the build. A Dockerfile is included in this repository, - which captures nearly all of the runtime dependencies needed to - execute the build. It is assumed that a Docker image will be - generated using this Dockerfile, and then the build executed in a - Docker container based on that Docker image. This way, the amount - of dependencies on the host system running the build is minimal. - - 2. The Docker host used to run the build must be based on Ubuntu 18.04. - As part of the build system, a ZFS pool and dataset will be - generated. The userspace ZFS utilities will be executed from the - Docker container, but they interact with the ZFS kernel modules - provided by the host. Thus, to ensure compatibility between the ZFS - userspace utilities in the Docker container, and the ZFS kernel - modules in the host, we require the host system to be running the - same Ubuntu release as the Docker container that will be used. - - 3. The Docker container must have access to Delphix's Artifactory - service, as well as Delphix's AWS S3 buckets; generally this is - accomplished by running the build within Delphix's VPN. This is - required so that the build can download Delphix's Java distribution - stored in Artifactory, along with the Delphix specific packages - stored in S3. + 1. Ansible must be installed and available to be used on the host + that'll run the build. An Ansible playbook is included in this + repository, which captures nearly all of the runtime dependencies + needed to execute the build. It is assumed that this playbook will + be used to configure the host that's used to execute the build. + + 2. The host that's used to execute the build must have access to + Delphix's Artifactory service, as well as Delphix's AWS S3 buckets; + generally this is accomplished by running the build within Delphix's + VPN. This is required so that the build can download Delphix's Java + distribution stored in Artifactory, along with the Delphix specific + packages stored in S3. ## Getting Started @@ -98,15 +87,7 @@ correcting any deficencies that may exist. This is easily done like so: Now, with the "bootstrap" VM properly configured, we can run the build: - $ ./scripts/docker-run.sh gradle ... - -This will create a new container based on the image we previously -created, and then execute "gradle" inside of that container. - -The "./scripts/docker-run" script can also be run without any arguments, -which will provide an interactive shell running in the container -environment, with the appliance-build git repository mounted inside of -the container; this can be useful for debugging and/or experimenting. + $ sudo ./gradlew ... Each variant will have ansible roles applied according to playbooks in per variant directories under live-build/variants. An appliance can be @@ -115,7 +96,7 @@ The task name has the form 'build\\'. For instance, the task to build the 'internal-minimal' variant for KVM is 'buildInternalMinimalKvm': - $ ./scripts/docker-run.sh gradle buildInternalMinimalKvm + $ sudo ./gradlew buildInternalMinimalKvm When this completes, the newly built VM artifacts will be contained in the "live-build/build/artifacts/" directory: @@ -156,7 +137,7 @@ An upgrade image for a particular variant can be built by running the an upgrade image for the internal-minimal variant is 'buildUpgradeImageInternalMinimal': - $ DELPHIX_PLATFORMS='kvm aws' ./scripts/docker-run.sh gradle buildUpgradeImageInternalMinimal + $ sudo DELPHIX_PLATFORMS='kvm aws' ./gradlew buildUpgradeImageInternalMinimal An upgrade image can contain the necessary packages to upgrade appliances running on multiple different platforms. Which platforms are @@ -182,7 +163,7 @@ Gradle. The most commonly used tasks are likely to be The complete list of tasks can be listed using the 'tasks' task: - $ ./scripts/docker-run.sh gradle tasks + $ ./gradlew tasks ## Creating new build variants @@ -253,7 +234,7 @@ For this example, we add our new role to the playboodk as shown below: See the instructions [above](#step-4-run-live-build) to setup your build environment and kick off the build: - $ ./scripts/docker-run.sh gradle buildInternalDcenterEsx + $ sudo ./gradlew buildInternalDcenterEsx ## Contributing diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 9587af38..40a77d34 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -1,5 +1,5 @@ # -# Copyright 2018 Delphix +# Copyright 2018-2019 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,19 +23,43 @@ - apt: name: - - docker.io - - python3-docker + - ansible + - aptly + - awscli + - bc + - coreutils + - equivs + - gdisk + - git + - java-package + - jq + - kpartx + - libxt6 + - livecd-rootfs + - make + - man + - openjdk-8-jre-headless + - pigz - qemu + - rename + - shellcheck + - vim - zfsutils-linux state: present +- modprobe: + name: zfs + state: present + +- apt: + name: + - docker.io + - python3-docker + state: present + # # We can't use the docker_image module because it doesn't yet support passing # the 'network' parameter: https://github.com/ansible/ansible/pull/50313, which # we need to be able to fetch things from Artifactory. # - shell: docker build --network host --tag "appliance-build:latest" "{{ toplevel.stdout }}/docker" - -- modprobe: - name: zfs - state: present diff --git a/bors.toml b/bors.toml deleted file mode 100644 index cef719e9..00000000 --- a/bors.toml +++ /dev/null @@ -1,16 +0,0 @@ -# -# The automation associated with the "jenkins/appliance-build" commit -# status uses a timeout of 12 hours. Thus, the bors timeout should match -# that. We then add an additional hour to the bors timeout, to account -# for the time it may take for the automation to start its testing. -# -timeout-sec = 46800 - -pr_status = [ - "license/cla" -] - -status = [ - "continuous-integration/travis-ci/push", - "jenkins/appliance-build" -] diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..87b738cbd051603d91cc39de6cb000dd98fe6b02 GIT binary patch literal 55190 zcmafaW0WS*vSoFbZQHhO+s0S6%`V%vZQJa!ZQHKus_B{g-pt%P_q|ywBQt-*Stldc z$+IJ3?^KWm27v+sf`9-50uuadKtMnL*BJ;1^6ynvR7H?hQcjE>7)art9Bu0Pcm@7C z@c%WG|JzYkP)<@zR9S^iR_sA`azaL$mTnGKnwDyMa;8yL_0^>Ba^)phg0L5rOPTbm7g*YIRLg-2^{qe^`rb!2KqS zk~5wEJtTdD?)3+}=eby3x6%i)sb+m??NHC^u=tcG8p$TzB<;FL(WrZGV&cDQb?O0GMe6PBV=V z?tTO*5_HTW$xea!nkc~Cnx#cL_rrUGWPRa6l+A{aiMY=<0@8y5OC#UcGeE#I>nWh}`#M#kIn-$A;q@u-p71b#hcSItS!IPw?>8 zvzb|?@Ahb22L(O4#2Sre&l9H(@TGT>#Py)D&eW-LNb!=S;I`ZQ{w;MaHW z#to!~TVLgho_Pm%zq@o{K3Xq?I|MVuVSl^QHnT~sHlrVxgsqD-+YD?Nz9@HA<;x2AQjxP)r6Femg+LJ-*)k%EZ}TTRw->5xOY z9#zKJqjZgC47@AFdk1$W+KhTQJKn7e>A&?@-YOy!v_(}GyV@9G#I?bsuto4JEp;5|N{orxi_?vTI4UF0HYcA( zKyGZ4<7Fk?&LZMQb6k10N%E*$gr#T&HsY4SPQ?yerqRz5c?5P$@6dlD6UQwZJ*Je9 z7n-@7!(OVdU-mg@5$D+R%gt82Lt%&n6Yr4=|q>XT%&^z_D*f*ug8N6w$`woqeS-+#RAOfSY&Rz z?1qYa5xi(7eTCrzCFJfCxc%j{J}6#)3^*VRKF;w+`|1n;Xaojr2DI{!<3CaP`#tXs z*`pBQ5k@JLKuCmovFDqh_`Q;+^@t_;SDm29 zCNSdWXbV?9;D4VcoV`FZ9Ggrr$i<&#Dx3W=8>bSQIU_%vf)#(M2Kd3=rN@^d=QAtC zI-iQ;;GMk|&A++W5#hK28W(YqN%?!yuW8(|Cf`@FOW5QbX|`97fxmV;uXvPCqxBD zJ9iI37iV)5TW1R+fV16y;6}2tt~|0J3U4E=wQh@sx{c_eu)t=4Yoz|%Vp<#)Qlh1V z0@C2ZtlT>5gdB6W)_bhXtcZS)`9A!uIOa`K04$5>3&8An+i9BD&GvZZ=7#^r=BN=k za+=Go;qr(M)B~KYAz|<^O3LJON}$Q6Yuqn8qu~+UkUKK~&iM%pB!BO49L+?AL7N7o z(OpM(C-EY753=G=WwJHE`h*lNLMNP^c^bBk@5MyP5{v7x>GNWH>QSgTe5 z!*GPkQ(lcbEs~)4ovCu!Zt&$${9$u(<4@9%@{U<-ksAqB?6F`bQ;o-mvjr)Jn7F&j$@`il1Mf+-HdBs<-`1FahTxmPMMI)@OtI&^mtijW6zGZ67O$UOv1Jj z;a3gmw~t|LjPkW3!EZ=)lLUhFzvO;Yvj9g`8hm%6u`;cuek_b-c$wS_0M4-N<@3l|88 z@V{Sd|M;4+H6guqMm4|v=C6B7mlpP(+It%0E;W`dxMOf9!jYwWj3*MRk`KpS_jx4c z=hrKBkFK;gq@;wUV2eqE3R$M+iUc+UD0iEl#-rECK+XmH9hLKrC={j@uF=f3UiceB zU5l$FF7#RKjx+6!JHMG5-!@zI-eG=a-!Bs^AFKqN_M26%cIIcSs61R$yuq@5a3c3& z4%zLs!g}+C5%`ja?F`?5-og0lv-;(^e<`r~p$x%&*89_Aye1N)9LNVk?9BwY$Y$$F^!JQAjBJvywXAesj7lTZ)rXuxv(FFNZVknJha99lN=^h`J2> zl5=~(tKwvHHvh|9-41@OV`c;Ws--PE%{7d2sLNbDp;A6_Ka6epzOSFdqb zBa0m3j~bT*q1lslHsHqaHIP%DF&-XMpCRL(v;MV#*>mB^&)a=HfLI7efblG z(@hzN`|n+oH9;qBklb=d^S0joHCsArnR1-h{*dIUThik>ot^!6YCNjg;J_i3h6Rl0ji)* zo(tQ~>xB!rUJ(nZjCA^%X;)H{@>uhR5|xBDA=d21p@iJ!cH?+%U|VSh2S4@gv`^)^ zNKD6YlVo$%b4W^}Rw>P1YJ|fTb$_(7C;hH+ z1XAMPb6*p^h8)e5nNPKfeAO}Ik+ZN_`NrADeeJOq4Ak;sD~ zTe77no{Ztdox56Xi4UE6S7wRVxJzWxKj;B%v7|FZ3cV9MdfFp7lWCi+W{}UqekdpH zdO#eoOuB3Fu!DU`ErfeoZWJbWtRXUeBzi zBTF-AI7yMC^ntG+8%mn(I6Dw}3xK8v#Ly{3w3_E?J4(Q5JBq~I>u3!CNp~Ekk&YH` z#383VO4O42NNtcGkr*K<+wYZ>@|sP?`AQcs5oqX@-EIqgK@Pmp5~p6O6qy4ml~N{D z{=jQ7k(9!CM3N3Vt|u@%ssTw~r~Z(}QvlROAkQQ?r8OQ3F0D$aGLh zny+uGnH5muJ<67Z=8uilKvGuANrg@s3Vu_lU2ajb?rIhuOd^E@l!Kl0hYIxOP1B~Q zggUmXbh$bKL~YQ#!4fos9UUVG#}HN$lIkM<1OkU@r>$7DYYe37cXYwfK@vrHwm;pg zbh(hEU|8{*d$q7LUm+x&`S@VbW*&p-sWrplWnRM|I{P;I;%U`WmYUCeJhYc|>5?&& zj}@n}w~Oo=l}iwvi7K6)osqa;M8>fRe}>^;bLBrgA;r^ZGgY@IC^ioRmnE&H4)UV5 zO{7egQ7sBAdoqGsso5q4R(4$4Tjm&&C|7Huz&5B0wXoJzZzNc5Bt)=SOI|H}+fbit z-PiF5(NHSy>4HPMrNc@SuEMDuKYMQ--G+qeUPqO_9mOsg%1EHpqoX^yNd~~kbo`cH zlV0iAkBFTn;rVb>EK^V6?T~t~3vm;csx+lUh_%ROFPy0(omy7+_wYjN!VRDtwDu^h4n|xpAMsLepm% zggvs;v8+isCW`>BckRz1MQ=l>K6k^DdT`~sDXTWQ<~+JtY;I~I>8XsAq3yXgxe>`O zZdF*{9@Z|YtS$QrVaB!8&`&^W->_O&-JXn1n&~}o3Z7FL1QE5R*W2W@=u|w~7%EeC1aRfGtJWxImfY-D3t!!nBkWM> zafu>^Lz-ONgT6ExjV4WhN!v~u{lt2-QBN&UxwnvdH|I%LS|J-D;o>@@sA62@&yew0 z)58~JSZP!(lX;da!3`d)D1+;K9!lyNlkF|n(UduR-%g>#{`pvrD^ClddhJyfL7C-(x+J+9&7EsC~^O`&}V%)Ut8^O_7YAXPDpzv8ir4 zl`d)(;imc6r16k_d^)PJZ+QPxxVJS5e^4wX9D=V2zH&wW0-p&OJe=}rX`*->XT=;_qI&)=WHkYnZx6bLoUh_)n-A}SF_ z9z7agNTM5W6}}ui=&Qs@pO5$zHsOWIbd_&%j^Ok5PJ3yUWQw*i4*iKO)_er2CDUME ztt+{Egod~W-fn^aLe)aBz)MOc_?i-stTj}~iFk7u^-gGSbU;Iem06SDP=AEw9SzuF zeZ|hKCG3MV(z_PJg0(JbqTRf4T{NUt%kz&}4S`)0I%}ZrG!jgW2GwP=WTtkWS?DOs znI9LY!dK+1_H0h+i-_~URb^M;4&AMrEO_UlDV8o?E>^3x%ZJyh$JuDMrtYL8|G3If zPf2_Qb_W+V?$#O; zydKFv*%O;Y@o_T_UAYuaqx1isMKZ^32JtgeceA$0Z@Ck0;lHbS%N5)zzAW9iz; z8tTKeK7&qw!8XVz-+pz>z-BeIzr*#r0nB^cntjQ9@Y-N0=e&ZK72vlzX>f3RT@i7@ z=z`m7jNk!9%^xD0ug%ptZnM>F;Qu$rlwo}vRGBIymPL)L|x}nan3uFUw(&N z24gdkcb7!Q56{0<+zu zEtc5WzG2xf%1<@vo$ZsuOK{v9gx^0`gw>@h>ZMLy*h+6ueoie{D#}}` zK2@6Xxq(uZaLFC%M!2}FX}ab%GQ8A0QJ?&!vaI8Gv=vMhd);6kGguDmtuOElru()) zuRk&Z{?Vp!G~F<1#s&6io1`poBqpRHyM^p;7!+L??_DzJ8s9mYFMQ0^%_3ft7g{PD zZd}8E4EV}D!>F?bzcX=2hHR_P`Xy6?FOK)mCj)Ym4s2hh z0OlOdQa@I;^-3bhB6mpw*X5=0kJv8?#XP~9){G-+0ST@1Roz1qi8PhIXp1D$XNqVG zMl>WxwT+K`SdO1RCt4FWTNy3!i?N>*-lbnn#OxFJrswgD7HjuKpWh*o@QvgF&j+CT z{55~ZsUeR1aB}lv#s_7~+9dCix!5(KR#c?K?e2B%P$fvrsZxy@GP#R#jwL{y#Ld$} z7sF>QT6m|}?V;msb?Nlohj7a5W_D$y+4O6eI;Zt$jVGymlzLKscqer9#+p2$0It&u zWY!dCeM6^B^Z;ddEmhi?8`scl=Lhi7W%2|pT6X6^%-=q90DS(hQ-%c+E*ywPvmoF(KqDoW4!*gmQIklm zk#!GLqv|cs(JRF3G?=AYY19{w@~`G3pa z@xR9S-Hquh*&5Yas*VI};(%9%PADn`kzm zeWMJVW=>>wap*9|R7n#!&&J>gq04>DTCMtj{P^d12|2wXTEKvSf?$AvnE!peqV7i4 zE>0G%CSn%WCW1yre?yi9*aFP{GvZ|R4JT}M%x_%Hztz2qw?&28l&qW<6?c6ym{f$d z5YCF+k#yEbjCN|AGi~-NcCG8MCF1!MXBFL{#7q z)HO+WW173?kuI}^Xat;Q^gb4Hi0RGyB}%|~j8>`6X4CPo+|okMbKy9PHkr58V4bX6<&ERU)QlF8%%huUz&f+dwTN|tk+C&&o@Q1RtG`}6&6;ncQuAcfHoxd5AgD7`s zXynq41Y`zRSiOY@*;&1%1z>oNcWTV|)sjLg1X8ijg1Y zbIGL0X*Sd}EXSQ2BXCKbJmlckY(@EWn~Ut2lYeuw1wg?hhj@K?XB@V_ZP`fyL~Yd3n3SyHU-RwMBr6t-QWE5TinN9VD4XVPU; zonIIR!&pGqrLQK)=#kj40Im%V@ij0&Dh0*s!lnTw+D`Dt-xmk-jmpJv$1-E-vfYL4 zqKr#}Gm}~GPE+&$PI@4ag@=M}NYi7Y&HW82Q`@Y=W&PE31D110@yy(1vddLt`P%N^ z>Yz195A%tnt~tvsSR2{m!~7HUc@x<&`lGX1nYeQUE(%sphTi>JsVqSw8xql*Ys@9B z>RIOH*rFi*C`ohwXjyeRBDt8p)-u{O+KWP;$4gg||%*u{$~yEj+Al zE(hAQRQ1k7MkCq9s4^N3ep*$h^L%2Vq?f?{+cicpS8lo)$Cb69b98au+m2J_e7nYwID0@`M9XIo1H~|eZFc8Hl!qly612ADCVpU zY8^*RTMX(CgehD{9v|^9vZ6Rab`VeZ2m*gOR)Mw~73QEBiktViBhR!_&3l$|be|d6 zupC`{g89Y|V3uxl2!6CM(RNpdtynaiJ~*DqSTq9Mh`ohZnb%^3G{k;6%n18$4nAqR zjPOrP#-^Y9;iw{J@XH9=g5J+yEVh|e=4UeY<^65`%gWtdQ=-aqSgtywM(1nKXh`R4 zzPP&7r)kv_uC7X9n=h=!Zrf<>X=B5f<9~Q>h#jYRD#CT7D~@6@RGNyO-#0iq0uHV1 zPJr2O4d_xLmg2^TmG7|dpfJ?GGa`0|YE+`2Rata9!?$j#e9KfGYuLL(*^z z!SxFA`$qm)q-YKh)WRJZ@S+-sD_1E$V?;(?^+F3tVcK6 z2fE=8hV*2mgiAbefU^uvcM?&+Y&E}vG=Iz!%jBF7iv){lyC`)*yyS~D8k+Mx|N3bm zI~L~Z$=W9&`x)JnO;8c>3LSDw!fzN#X3qi|0`sXY4?cz{*#xz!kvZ9bO=K3XbN z5KrgN=&(JbXH{Wsu9EdmQ-W`i!JWEmfI;yVTT^a-8Ch#D8xf2dtyi?7p z%#)W3n*a#ndFpd{qN|+9Jz++AJQO#-Y7Z6%*%oyEP5zs}d&kKIr`FVEY z;S}@d?UU=tCdw~EJ{b}=9x}S2iv!!8<$?d7VKDA8h{oeD#S-$DV)-vPdGY@x08n)@ zag?yLF_E#evvRTj4^CcrLvBL=fft&@HOhZ6Ng4`8ijt&h2y}fOTC~7GfJi4vpomA5 zOcOM)o_I9BKz}I`q)fu+Qnfy*W`|mY%LO>eF^a z;$)?T4F-(X#Q-m}!-k8L_rNPf`Mr<9IWu)f&dvt=EL+ESYmCvErd@8B9hd)afc(ZL94S z?rp#h&{7Ah5IJftK4VjATklo7@hm?8BX*~oBiz)jyc9FuRw!-V;Uo>p!CWpLaIQyt zAs5WN)1CCeux-qiGdmbIk8LR`gM+Qg=&Ve}w?zA6+sTL)abU=-cvU`3E?p5$Hpkxw znu0N659qR=IKnde*AEz_7z2pdi_Bh-sb3b=PdGO1Pdf_q2;+*Cx9YN7p_>rl``knY zRn%aVkcv1(W;`Mtp_DNOIECtgq%ufk-mu_<+Fu3Q17Tq4Rr(oeq)Yqk_CHA7LR@7@ zIZIDxxhS&=F2IQfusQ+Nsr%*zFK7S4g!U0y@3H^Yln|i;0a5+?RPG;ZSp6Tul>ezM z`40+516&719qT)mW|ArDSENle5hE2e8qY+zfeZoy12u&xoMgcP)4=&P-1Ib*-bAy` zlT?>w&B|ei-rCXO;sxo7*G;!)_p#%PAM-?m$JP(R%x1Hfas@KeaG%LO?R=lmkXc_MKZW}3f%KZ*rAN?HYvbu2L$ zRt_uv7~-IejlD1x;_AhwGXjB94Q=%+PbxuYzta*jw?S&%|qb=(JfJ?&6P=R7X zV%HP_!@-zO*zS}46g=J}#AMJ}rtWBr21e6hOn&tEmaM%hALH7nlm2@LP4rZ>2 zebe5aH@k!e?ij4Zwak#30|}>;`bquDQK*xmR=zc6vj0yuyC6+U=LusGnO3ZKFRpen z#pwzh!<+WBVp-!$MAc<0i~I%fW=8IO6K}bJ<-Scq>e+)951R~HKB?Mx2H}pxPHE@} zvqpq5j81_jtb_WneAvp<5kgdPKm|u2BdQx9%EzcCN&U{l+kbkhmV<1}yCTDv%&K^> zg;KCjwh*R1f_`6`si$h6`jyIKT7rTv5#k~x$mUyIw)_>Vr)D4fwIs@}{FSX|5GB1l z4vv;@oS@>Bu7~{KgUa_8eg#Lk6IDT2IY$41$*06{>>V;Bwa(-@N;ex4;D`(QK*b}{ z{#4$Hmt)FLqERgKz=3zXiV<{YX6V)lvYBr3V>N6ajeI~~hGR5Oe>W9r@sg)Na(a4- zxm%|1OKPN6^%JaD^^O~HbLSu=f`1px>RawOxLr+1b2^28U*2#h*W^=lSpSY4(@*^l z{!@9RSLG8Me&RJYLi|?$c!B0fP=4xAM4rerxX{xy{&i6=AqXueQAIBqO+pmuxy8Ib z4X^}r!NN3-upC6B#lt7&x0J;)nb9O~xjJMemm$_fHuP{DgtlU3xiW0UesTzS30L+U zQzDI3p&3dpONhd5I8-fGk^}@unluzu%nJ$9pzoO~Kk!>dLxw@M)M9?pNH1CQhvA`z zV;uacUtnBTdvT`M$1cm9`JrT3BMW!MNVBy%?@ZX%;(%(vqQAz<7I!hlDe|J3cn9=} zF7B;V4xE{Ss76s$W~%*$JviK?w8^vqCp#_G^jN0j>~Xq#Zru26e#l3H^{GCLEXI#n z?n~F-Lv#hU(bZS`EI9(xGV*jT=8R?CaK)t8oHc9XJ;UPY0Hz$XWt#QyLBaaz5+}xM zXk(!L_*PTt7gwWH*HLWC$h3Ho!SQ-(I||nn_iEC{WT3S{3V{8IN6tZ1C+DiFM{xlI zeMMk{o5;I6UvaC)@WKp9D+o?2Vd@4)Ue-nYci()hCCsKR`VD;hr9=vA!cgGL%3k^b(jADGyPi2TKr(JNh8mzlIR>n(F_hgiV(3@Ds(tjbNM7GoZ;T|3 zWzs8S`5PrA!9){jBJuX4y`f<4;>9*&NY=2Sq2Bp`M2(fox7ZhIDe!BaQUb@P(ub9D zlP8!p(AN&CwW!V&>H?yPFMJ)d5x#HKfwx;nS{Rr@oHqpktOg)%F+%1#tsPtq7zI$r zBo-Kflhq-=7_eW9B2OQv=@?|y0CKN77)N;z@tcg;heyW{wlpJ1t`Ap!O0`Xz{YHqO zI1${8Hag^r!kA<2_~bYtM=<1YzQ#GGP+q?3T7zYbIjN6Ee^V^b&9en$8FI*NIFg9G zPG$OXjT0Ku?%L7fat8Mqbl1`azf1ltmKTa(HH$Dqlav|rU{zP;Tbnk-XkGFQ6d+gi z-PXh?_kEJl+K98&OrmzgPIijB4!Pozbxd0H1;Usy!;V>Yn6&pu*zW8aYx`SC!$*ti zSn+G9p=~w6V(fZZHc>m|PPfjK6IN4(o=IFu?pC?+`UZAUTw!e`052{P=8vqT^(VeG z=psASIhCv28Y(;7;TuYAe>}BPk5Qg=8$?wZj9lj>h2kwEfF_CpK=+O6Rq9pLn4W)# zeXCKCpi~jsfqw7Taa0;!B5_C;B}e56W1s8@p*)SPzA;Fd$Slsn^=!_&!mRHV*Lmt| zBGIDPuR>CgS4%cQ4wKdEyO&Z>2aHmja;Pz+n|7(#l%^2ZLCix%>@_mbnyPEbyrHaz z>j^4SIv;ZXF-Ftzz>*t4wyq)ng8%0d;(Z_ExZ-cxwei=8{(br-`JYO(f23Wae_MqE z3@{Mlf^%M5G1SIN&en1*| zH~ANY1h3&WNsBy$G9{T=`kcxI#-X|>zLX2r*^-FUF+m0{k)n#GTG_mhG&fJfLj~K& zU~~6othMlvMm9<*SUD2?RD+R17|Z4mgR$L*R3;nBbo&Vm@39&3xIg;^aSxHS>}gwR zmzs?h8oPnNVgET&dx5^7APYx6Vv6eou07Zveyd+^V6_LzI$>ic+pxD_8s~ zC<}ucul>UH<@$KM zT4oI=62M%7qQO{}re-jTFqo9Z;rJKD5!X5$iwUsh*+kcHVhID08MB5cQD4TBWB(rI zuWc%CA}}v|iH=9gQ?D$1#Gu!y3o~p7416n54&Hif`U-cV?VrUMJyEqo_NC4#{puzU zzXEE@UppeeRlS9W*^N$zS`SBBi<@tT+<%3l@KhOy^%MWB9(A#*J~DQ;+MK*$rxo6f zcx3$3mcx{tly!q(p2DQrxcih|)0do_ZY77pyHGE#Q(0k*t!HUmmMcYFq%l$-o6%lS zDb49W-E?rQ#Hl``C3YTEdGZjFi3R<>t)+NAda(r~f1cT5jY}s7-2^&Kvo&2DLTPYP zhVVo-HLwo*vl83mtQ9)PR#VBg)FN}+*8c-p8j`LnNUU*Olm1O1Qqe62D#$CF#?HrM zy(zkX|1oF}Z=T#3XMLWDrm(|m+{1&BMxHY7X@hM_+cV$5-t!8HT(dJi6m9{ja53Yw z3f^`yb6Q;(e|#JQIz~B*=!-GbQ4nNL-NL z@^NWF_#w-Cox@h62;r^;Y`NX8cs?l^LU;5IWE~yvU8TqIHij!X8ydbLlT0gwmzS9} z@5BccG?vO;rvCs$mse1*ANi-cYE6Iauz$Fbn3#|ToAt5v7IlYnt6RMQEYLldva{~s zvr>1L##zmeoYgvIXJ#>bbuCVuEv2ZvZ8I~PQUN3wjP0UC)!U+wn|&`V*8?)` zMSCuvnuGec>QL+i1nCPGDAm@XSMIo?A9~C?g2&G8aNKjWd2pDX{qZ?04+2 zeyLw}iEd4vkCAWwa$ zbrHlEf3hfN7^1g~aW^XwldSmx1v~1z(s=1az4-wl} z`mM+G95*N*&1EP#u3}*KwNrPIgw8Kpp((rdEOO;bT1;6ea~>>sK+?!;{hpJ3rR<6UJb`O8P4@{XGgV%63_fs%cG8L zk9Fszbdo4tS$g0IWP1>t@0)E%-&9yj%Q!fiL2vcuL;90fPm}M==<>}Q)&sp@STFCY z^p!RzmN+uXGdtPJj1Y-khNyCb6Y$Vs>eZyW zPaOV=HY_T@FwAlleZCFYl@5X<<7%5DoO(7S%Lbl55?{2vIr_;SXBCbPZ(up;pC6Wx={AZL?shYOuFxLx1*>62;2rP}g`UT5+BHg(ju z&7n5QSvSyXbioB9CJTB#x;pexicV|9oaOpiJ9VK6EvKhl4^Vsa(p6cIi$*Zr0UxQ z;$MPOZnNae2Duuce~7|2MCfhNg*hZ9{+8H3?ts9C8#xGaM&sN;2lriYkn9W>&Gry! z3b(Xx1x*FhQkD-~V+s~KBfr4M_#0{`=Yrh90yj}Ph~)Nx;1Y^8<418tu!$1<3?T*~ z7Dl0P3Uok-7w0MPFQexNG1P5;y~E8zEvE49>$(f|XWtkW2Mj`udPn)pb%} zrA%wRFp*xvDgC767w!9`0vx1=q!)w!G+9(-w&p*a@WXg{?T&%;qaVcHo>7ca%KX$B z^7|KBPo<2;kM{2mRnF8vKm`9qGV%|I{y!pKm8B(q^2V;;x2r!1VJ^Zz8bWa)!-7a8 zSRf@dqEPlsj!7}oNvFFAA)75})vTJUwQ03hD$I*j6_5xbtd_JkE2`IJD_fQ;a$EkO z{fQ{~e%PKgPJsD&PyEvDmg+Qf&p*-qu!#;1k2r_(H72{^(Z)htgh@F?VIgK#_&eS- z$~(qInec>)XIkv@+{o6^DJLpAb>!d}l1DK^(l%#OdD9tKK6#|_R?-%0V!`<9Hj z3w3chDwG*SFte@>Iqwq`J4M&{aHXzyigT620+Vf$X?3RFfeTcvx_e+(&Q*z)t>c0e zpZH$1Z3X%{^_vylHVOWT6tno=l&$3 z9^eQ@TwU#%WMQaFvaYp_we%_2-9=o{+ck zF{cKJCOjpW&qKQquyp2BXCAP920dcrZ}T1@piukx_NY;%2W>@Wca%=Ch~x5Oj58Hv z;D-_ALOZBF(Mqbcqjd}P3iDbek#Dwzu`WRs`;hRIr*n0PV7vT+%Io(t}8KZ zpp?uc2eW!v28ipep0XNDPZt7H2HJ6oey|J3z!ng#1H~x_k%35P+Cp%mqXJ~cV0xdd z^4m5^K_dQ^Sg?$P`))ccV=O>C{Ds(C2WxX$LMC5vy=*44pP&)X5DOPYfqE${)hDg< z3hcG%U%HZ39=`#Ko4Uctg&@PQLf>?0^D|4J(_1*TFMOMB!Vv1_mnOq$BzXQdOGqgy zOp#LBZ!c>bPjY1NTXksZmbAl0A^Y&(%a3W-k>bE&>K?px5Cm%AT2E<&)Y?O*?d80d zgI5l~&Mve;iXm88Q+Fw7{+`PtN4G7~mJWR^z7XmYQ>uoiV!{tL)hp|= zS(M)813PM`d<501>{NqaPo6BZ^T{KBaqEVH(2^Vjeq zgeMeMpd*1tE@@);hGjuoVzF>Cj;5dNNwh40CnU+0DSKb~GEMb_# zT8Z&gz%SkHq6!;_6dQFYE`+b`v4NT7&@P>cA1Z1xmXy<2htaDhm@XXMp!g($ zw(7iFoH2}WR`UjqjaqOQ$ecNt@c|K1H1kyBArTTjLp%-M`4nzOhkfE#}dOpcd;b#suq8cPJ&bf5`6Tq>ND(l zib{VrPZ>{KuaIg}Y$W>A+nrvMg+l4)-@2jpAQ5h(Tii%Ni^-UPVg{<1KGU2EIUNGaXcEkOedJOusFT9X3%Pz$R+-+W+LlRaY-a$5r?4V zbPzgQl22IPG+N*iBRDH%l{Zh$fv9$RN1sU@Hp3m=M}{rX%y#;4(x1KR2yCO7Pzo>rw(67E{^{yUR`91nX^&MxY@FwmJJbyPAoWZ9Z zcBS$r)&ogYBn{DOtD~tIVJUiq|1foX^*F~O4hlLp-g;Y2wKLLM=?(r3GDqsPmUo*? zwKMEi*%f)C_@?(&&hk>;m07F$X7&i?DEK|jdRK=CaaNu-)pX>n3}@%byPKVkpLzBq z{+Py&!`MZ^4@-;iY`I4#6G@aWMv{^2VTH7|WF^u?3vsB|jU3LgdX$}=v7#EHRN(im zI(3q-eU$s~r=S#EWqa_2!G?b~ z<&brq1vvUTJH380=gcNntZw%7UT8tLAr-W49;9y^=>TDaTC|cKA<(gah#2M|l~j)w zY8goo28gj$n&zcNgqX1Qn6=<8?R0`FVO)g4&QtJAbW3G#D)uNeac-7cH5W#6i!%BH z=}9}-f+FrtEkkrQ?nkoMQ1o-9_b+&=&C2^h!&mWFga#MCrm85hW;)1pDt;-uvQG^D zntSB?XA*0%TIhtWDS!KcI}kp3LT>!(Nlc(lQN?k^bS8Q^GGMfo}^|%7s;#r+pybl@?KA++|FJ zr%se9(B|g*ERQU96az%@4gYrxRRxaM2*b}jNsG|0dQi;Rw{0WM0E>rko!{QYAJJKY z)|sX0N$!8d9E|kND~v|f>3YE|uiAnqbkMn)hu$if4kUkzKqoNoh8v|S>VY1EKmgO} zR$0UU2o)4i4yc1inx3}brso+sio{)gfbLaEgLahj8(_Z#4R-v) zglqwI%`dsY+589a8$Mu7#7_%kN*ekHupQ#48DIN^uhDxblDg3R1yXMr^NmkR z7J_NWCY~fhg}h!_aXJ#?wsZF$q`JH>JWQ9`jbZzOBpS`}-A$Vgkq7+|=lPx9H7QZG z8i8guMN+yc4*H*ANr$Q-3I{FQ-^;8ezWS2b8rERp9TMOLBxiG9J*g5=?h)mIm3#CGi4JSq1ohFrcrxx@`**K5%T}qbaCGldV!t zVeM)!U3vbf5FOy;(h08JnhSGxm)8Kqxr9PsMeWi=b8b|m_&^@#A3lL;bVKTBx+0v8 zLZeWAxJ~N27lsOT2b|qyp$(CqzqgW@tyy?CgwOe~^i;ZH zlL``i4r!>i#EGBNxV_P@KpYFQLz4Bdq{#zA&sc)*@7Mxsh9u%e6Ke`?5Yz1jkTdND zR8!u_yw_$weBOU}24(&^Bm|(dSJ(v(cBct}87a^X(v>nVLIr%%D8r|&)mi+iBc;B;x;rKq zd8*X`r?SZsTNCPQqoFOrUz8nZO?225Z#z(B!4mEp#ZJBzwd7jW1!`sg*?hPMJ$o`T zR?KrN6OZA1H{9pA;p0cSSu;@6->8aJm1rrO-yDJ7)lxuk#npUk7WNER1Wwnpy%u zF=t6iHzWU(L&=vVSSc^&D_eYP3TM?HN!Tgq$SYC;pSIPWW;zeNm7Pgub#yZ@7WPw#f#Kl)W4%B>)+8%gpfoH1qZ;kZ*RqfXYeGXJ_ zk>2otbp+1By`x^1V!>6k5v8NAK@T;89$`hE0{Pc@Q$KhG0jOoKk--Qx!vS~lAiypV zCIJ&6B@24`!TxhJ4_QS*S5;;Pk#!f(qIR7*(c3dN*POKtQe)QvR{O2@QsM%ujEAWEm) z+PM=G9hSR>gQ`Bv2(k}RAv2+$7qq(mU`fQ+&}*i%-RtSUAha>70?G!>?w%F(b4k!$ zvm;E!)2`I?etmSUFW7WflJ@8Nx`m_vE2HF#)_BiD#FaNT|IY@!uUbd4v$wTglIbIX zblRy5=wp)VQzsn0_;KdM%g<8@>#;E?vypTf=F?3f@SSdZ;XpX~J@l1;p#}_veWHp>@Iq_T z@^7|h;EivPYv1&u0~l9(a~>dV9Uw10QqB6Dzu1G~-l{*7IktljpK<_L8m0|7VV_!S zRiE{u97(%R-<8oYJ{molUd>vlGaE-C|^<`hppdDz<7OS13$#J zZ+)(*rZIDSt^Q$}CRk0?pqT5PN5TT`Ya{q(BUg#&nAsg6apPMhLTno!SRq1e60fl6GvpnwDD4N> z9B=RrufY8+g3_`@PRg+(+gs2(bd;5#{uTZk96CWz#{=&h9+!{_m60xJxC%r&gd_N! z>h5UzVX%_7@CUeAA1XFg_AF%(uS&^1WD*VPS^jcC!M2v@RHZML;e(H-=(4(3O&bX- zI6>usJOS+?W&^S&DL{l|>51ZvCXUKlH2XKJPXnHjs*oMkNM#ZDLx!oaM5(%^)5XaP zk6&+P16sA>vyFe9v`Cp5qnbE#r#ltR5E+O3!WnKn`56Grs2;sqr3r# zp@Zp<^q`5iq8OqOlJ`pIuyK@3zPz&iJ0Jcc`hDQ1bqos2;}O|$i#}e@ua*x5VCSx zJAp}+?Hz++tm9dh3Fvm_bO6mQo38al#>^O0g)Lh^&l82+&x)*<n7^Sw-AJo9tEzZDwyJ7L^i7|BGqHu+ea6(&7jKpBq>~V z8CJxurD)WZ{5D0?s|KMi=e7A^JVNM6sdwg@1Eg_+Bw=9j&=+KO1PG|y(mP1@5~x>d z=@c{EWU_jTSjiJl)d(>`qEJ;@iOBm}alq8;OK;p(1AdH$)I9qHNmxxUArdzBW0t+Qeyl)m3?D09770g z)hzXEOy>2_{?o%2B%k%z4d23!pZcoxyW1Ik{|m7Q1>fm4`wsRrl)~h z_=Z*zYL+EG@DV1{6@5@(Ndu!Q$l_6Qlfoz@79q)Kmsf~J7t1)tl#`MD<;1&CAA zH8;i+oBm89dTTDl{aH`cmTPTt@^K-%*sV+t4X9q0Z{A~vEEa!&rRRr=0Rbz4NFCJr zLg2u=0QK@w9XGE=6(-JgeP}G#WG|R&tfHRA3a9*zh5wNTBAD;@YYGx%#E4{C#Wlfo z%-JuW9=FA_T6mR2-Vugk1uGZvJbFvVVWT@QOWz$;?u6+CbyQsbK$>O1APk|xgnh_8 zc)s@Mw7#0^wP6qTtyNq2G#s?5j~REyoU6^lT7dpX{T-rhZWHD%dik*=EA7bIJgOVf_Ga!yC8V^tkTOEHe+JK@Fh|$kfNxO^= z#lpV^(ZQ-3!^_BhV>aXY~GC9{8%1lOJ}6vzXDvPhC>JrtXwFBC+!3a*Z-%#9}i z#<5&0LLIa{q!rEIFSFc9)>{-_2^qbOg5;_A9 ztQ))C6#hxSA{f9R3Eh^`_f${pBJNe~pIQ`tZVR^wyp}=gLK}e5_vG@w+-mp#Fu>e| z*?qBp5CQ5zu+Fi}xAs)YY1;bKG!htqR~)DB$ILN6GaChoiy%Bq@i+1ZnANC0U&D z_4k$=YP47ng+0NhuEt}6C;9-JDd8i5S>`Ml==9wHDQFOsAlmtrVwurYDw_)Ihfk35 zJDBbe!*LUpg%4n>BExWz>KIQ9vexUu^d!7rc_kg#Bf= z7TLz|l*y*3d2vi@c|pX*@ybf!+Xk|2*z$@F4K#MT8Dt4zM_EcFmNp31#7qT6(@GG? zdd;sSY9HHuDb=w&|K%sm`bYX#%UHKY%R`3aLMO?{T#EI@FNNFNO>p@?W*i0z(g2dt z{=9Ofh80Oxv&)i35AQN>TPMjR^UID-T7H5A?GI{MD_VeXZ%;uo41dVm=uT&ne2h0i zv*xI%9vPtdEK@~1&V%p1sFc2AA`9?H)gPnRdlO~URx!fiSV)j?Tf5=5F>hnO=$d$x zzaIfr*wiIc!U1K*$JO@)gP4%xp!<*DvJSv7p}(uTLUb=MSb@7_yO+IsCj^`PsxEl& zIxsi}s3L?t+p+3FXYqujGhGwTx^WXgJ1}a@Yq5mwP0PvGEr*qu7@R$9j>@-q1rz5T zriz;B^(ex?=3Th6h;7U`8u2sDlfS{0YyydK=*>-(NOm9>S_{U|eg(J~C7O zIe{|LK=Y`hXiF_%jOM8Haw3UtaE{hWdzo3BbD6ud7br4cODBtN(~Hl+odP0SSWPw;I&^m)yLw+nd#}3#z}?UIcX3=SssI}`QwY=% zAEXTODk|MqTx}2DVG<|~(CxgLyi*A{m>M@1h^wiC)4Hy>1K7@|Z&_VPJsaQoS8=ex zDL&+AZdQa>ylxhT_Q$q=60D5&%pi6+qlY3$3c(~rsITX?>b;({FhU!7HOOhSP7>bmTkC8KM%!LRGI^~y3Ug+gh!QM=+NZXznM)?L3G=4=IMvFgX3BAlyJ z`~jjA;2z+65D$j5xbv9=IWQ^&-K3Yh`vC(1Qz2h2`o$>Cej@XRGff!it$n{@WEJ^N z41qk%Wm=}mA*iwCqU_6}Id!SQd13aFER3unXaJJXIsSnxvG2(hSCP{i&QH$tL&TPx zDYJsuk+%laN&OvKb-FHK$R4dy%M7hSB*yj#-nJy?S9tVoxAuDei{s}@+pNT!vLOIC z8g`-QQW8FKp3cPsX%{)0B+x+OhZ1=L7F-jizt|{+f1Ga7%+!BXqjCjH&x|3%?UbN# zh?$I1^YokvG$qFz5ySK+Ja5=mkR&p{F}ev**rWdKMko+Gj^?Or=UH?SCg#0F(&a_y zXOh}dPv0D9l0RVedq1~jCNV=8?vZfU-Xi|nkeE->;ohG3U7z+^0+HV17~-_Mv#mV` zzvwUJJ15v5wwKPv-)i@dsEo@#WEO9zie7mdRAbgL2kjbW4&lk$vxkbq=w5mGKZK6@ zjXWctDkCRx58NJD_Q7e}HX`SiV)TZMJ}~zY6P1(LWo`;yDynY_5_L?N-P`>ALfmyl z8C$a~FDkcwtzK9m$tof>(`Vu3#6r#+v8RGy#1D2)F;vnsiL&P-c^PO)^B-4VeJteLlT@25sPa z%W~q5>YMjj!mhN})p$47VA^v$Jo6_s{!y?}`+h+VM_SN`!11`|;C;B};B&Z<@%FOG z_YQVN+zFF|q5zKab&e4GH|B;sBbKimHt;K@tCH+S{7Ry~88`si7}S)1E{21nldiu5 z_4>;XTJa~Yd$m4A9{Qbd)KUAm7XNbZ4xHbg3a8-+1uf*$1PegabbmCzgC~1WB2F(W zYj5XhVos!X!QHuZXCatkRsdEsSCc+D2?*S7a+(v%toqyxhjz|`zdrUvsxQS{J>?c& zvx*rHw^8b|v^7wq8KWVofj&VUitbm*a&RU_ln#ZFA^3AKEf<#T%8I!Lg3XEsdH(A5 zlgh&M_XEoal)i#0tcq8c%Gs6`xu;vvP2u)D9p!&XNt z!TdF_H~;`g@fNXkO-*t<9~;iEv?)Nee%hVe!aW`N%$cFJ(Dy9+Xk*odyFj72T!(b%Vo5zvCGZ%3tkt$@Wcx8BWEkefI1-~C_3y*LjlQ5%WEz9WD8i^ z2MV$BHD$gdPJV4IaV)G9CIFwiV=ca0cfXdTdK7oRf@lgyPx;_7*RRFk=?@EOb9Gcz zg~VZrzo*Snp&EE{$CWr)JZW)Gr;{B2ka6B!&?aknM-FENcl%45#y?oq9QY z3^1Y5yn&^D67Da4lI}ljDcphaEZw2;tlYuzq?uB4b9Mt6!KTW&ptxd^vF;NbX=00T z@nE1lIBGgjqs?ES#P{ZfRb6f!At51vk%<0X%d_~NL5b8UyfQMPDtfU@>ijA0NP3UU zh{lCf`Wu7cX!go`kUG`1K=7NN@SRGjUKuo<^;@GS!%iDXbJs`o6e`v3O8-+7vRkFm z)nEa$sD#-v)*Jb>&Me+YIW3PsR1)h=-Su)))>-`aRcFJG-8icomO4J@60 zw10l}BYxi{eL+Uu0xJYk-Vc~BcR49Qyyq!7)PR27D`cqGrik=?k1Of>gY7q@&d&Ds zt7&WixP`9~jjHO`Cog~RA4Q%uMg+$z^Gt&vn+d3&>Ux{_c zm|bc;k|GKbhZLr-%p_f%dq$eiZ;n^NxoS-Nu*^Nx5vm46)*)=-Bf<;X#?`YC4tLK; z?;u?shFbXeks+dJ?^o$l#tg*1NA?(1iFff@I&j^<74S!o;SWR^Xi);DM%8XiWpLi0 zQE2dL9^a36|L5qC5+&Pf0%>l&qQ&)OU4vjd)%I6{|H+pw<0(a``9w(gKD&+o$8hOC zNAiShtc}e~ob2`gyVZx59y<6Fpl*$J41VJ-H*e-yECWaDMmPQi-N8XI3 z%iI@ljc+d}_okL1CGWffeaejlxWFVDWu%e=>H)XeZ|4{HlbgC-Uvof4ISYQzZ0Um> z#Ov{k1c*VoN^f(gfiueuag)`TbjL$XVq$)aCUBL_M`5>0>6Ska^*Knk__pw{0I>jA zzh}Kzg{@PNi)fcAk7jMAdi-_RO%x#LQszDMS@_>iFoB+zJ0Q#CQJzFGa8;pHFdi`^ zxnTC`G$7Rctm3G8t8!SY`GwFi4gF|+dAk7rh^rA{NXzc%39+xSYM~($L(pJ(8Zjs* zYdN_R^%~LiGHm9|ElV4kVZGA*T$o@YY4qpJOxGHlUi*S*A(MrgQ{&xoZQo+#PuYRs zv3a$*qoe9gBqbN|y|eaH=w^LE{>kpL!;$wRahY(hhzRY;d33W)m*dfem@)>pR54Qy z ze;^F?mwdU?K+=fBabokSls^6_6At#1Sh7W*y?r6Ss*dmZP{n;VB^LDxM1QWh;@H0J z!4S*_5j_;+@-NpO1KfQd&;C7T`9ak;X8DTRz$hDNcjG}xAfg%gwZSb^zhE~O);NMO zn2$fl7Evn%=Lk!*xsM#(y$mjukN?A&mzEw3W5>_o+6oh62kq=4-`e3B^$rG=XG}Kd zK$blh(%!9;@d@3& zGFO60j1Vf54S}+XD?%*uk7wW$f`4U3F*p7@I4Jg7f`Il}2H<{j5h?$DDe%wG7jZQL zI{mj?t?Hu>$|2UrPr5&QyK2l3mas?zzOk0DV30HgOQ|~xLXDQ8M3o#;CNKO8RK+M; zsOi%)js-MU>9H4%Q)#K_me}8OQC1u;f4!LO%|5toa1|u5Q@#mYy8nE9IXmR}b#sZK z3sD395q}*TDJJA9Er7N`y=w*S&tA;mv-)Sx4(k$fJBxXva0_;$G6!9bGBw13c_Uws zXks4u(8JA@0O9g5f?#V~qR5*u5aIe2HQO^)RW9TTcJk28l`Syl>Q#ZveEE4Em+{?%iz6=V3b>rCm9F zPQQm@-(hfNdo2%n?B)u_&Qh7^^@U>0qMBngH8}H|v+Ejg*Dd(Y#|jgJ-A zQ_bQscil%eY}8oN7ZL+2r|qv+iJY?*l)&3W_55T3GU;?@Om*(M`u0DXAsQ7HSl56> z4P!*(%&wRCb?a4HH&n;lAmr4rS=kMZb74Akha2U~Ktni>>cD$6jpugjULq)D?ea%b zk;UW0pAI~TH59P+o}*c5Ei5L-9OE;OIBt>^(;xw`>cN2`({Rzg71qrNaE=cAH^$wP zNrK9Glp^3a%m+ilQj0SnGq`okjzmE7<3I{JLD6Jn^+oas=h*4>Wvy=KXqVBa;K&ri z4(SVmMXPG}0-UTwa2-MJ=MTfM3K)b~DzSVq8+v-a0&Dsv>4B65{dBhD;(d44CaHSM zb!0ne(*<^Q%|nuaL`Gb3D4AvyO8wyygm=1;9#u5x*k0$UOwx?QxR*6Od8>+ujfyo0 zJ}>2FgW_iv(dBK2OWC-Y=Tw!UwIeOAOUUC;h95&S1hn$G#if+d;*dWL#j#YWswrz_ zMlV=z+zjZJ%SlDhxf)vv@`%~$Afd)T+MS1>ZE7V$Rj#;J*<9Ld=PrK0?qrazRJWx) z(BTLF@Wk279nh|G%ZY7_lK7=&j;x`bMND=zgh_>>-o@6%8_#Bz!FnF*onB@_k|YCF z?vu!s6#h9bL3@tPn$1;#k5=7#s*L;FLK#=M89K^|$3LICYWIbd^qguQp02w5>8p-H z+@J&+pP_^iF4Xu>`D>DcCnl8BUwwOlq6`XkjHNpi@B?OOd`4{dL?kH%lt78(-L}eah8?36zw9d-dI6D{$s{f=M7)1 zRH1M*-82}DoFF^Mi$r}bTB5r6y9>8hjL54%KfyHxn$LkW=AZ(WkHWR;tIWWr@+;^^ zVomjAWT)$+rn%g`LHB6ZSO@M3KBA? z+W7ThSBgpk`jZHZUrp`F;*%6M5kLWy6AW#T{jFHTiKXP9ITrMlEdti7@&AT_a-BA!jc(Kt zWk>IdY-2Zbz?U1)tk#n_Lsl?W;0q`;z|t9*g-xE!(}#$fScX2VkjSiboKWE~afu5d z2B@9mvT=o2fB_>Mnie=TDJB+l`GMKCy%2+NcFsbpv<9jS@$X37K_-Y!cvF5NEY`#p z3sWEc<7$E*X*fp+MqsOyMXO=<2>o8)E(T?#4KVQgt=qa%5FfUG_LE`n)PihCz2=iNUt7im)s@;mOc9SR&{`4s9Q6)U31mn?}Y?$k3kU z#h??JEgH-HGt`~%)1ZBhT9~uRi8br&;a5Y3K_Bl1G)-y(ytx?ok9S*Tz#5Vb=P~xH z^5*t_R2It95=!XDE6X{MjLYn4Eszj9Y91T2SFz@eYlx9Z9*hWaS$^5r7=W5|>sY8}mS(>e9Ez2qI1~wtlA$yv2e-Hjn&K*P z2zWSrC~_8Wrxxf#%QAL&f8iH2%R)E~IrQLgWFg8>`Vnyo?E=uiALoRP&qT{V2{$79 z%9R?*kW-7b#|}*~P#cA@q=V|+RC9=I;aK7Pju$K-n`EoGV^-8Mk=-?@$?O37evGKn z3NEgpo_4{s>=FB}sqx21d3*=gKq-Zk)U+bM%Q_}0`XGkYh*+jRaP+aDnRv#Zz*n$pGp zEU9omuYVXH{AEx>=kk}h2iKt!yqX=EHN)LF}z1j zJx((`CesN1HxTFZ7yrvA2jTPmKYVij>45{ZH2YtsHuGzIRotIFj?(8T@ZWUv{_%AI zgMZlB03C&FtgJqv9%(acqt9N)`4jy4PtYgnhqev!r$GTIOvLF5aZ{tW5MN@9BDGu* zBJzwW3sEJ~Oy8is`l6Ly3an7RPtRr^1Iu(D!B!0O241Xua>Jee;Rc7tWvj!%#yX#m z&pU*?=rTVD7pF6va1D@u@b#V@bShFr3 zMyMbNCZwT)E-%L-{%$3?n}>EN>ai7b$zR_>=l59mW;tfKj^oG)>_TGCJ#HbLBsNy$ zqAqPagZ3uQ(Gsv_-VrZmG&hHaOD#RB#6J8&sL=^iMFB=gH5AIJ+w@sTf7xa&Cnl}@ zxrtzoNq>t?=(+8bS)s2p3>jW}tye0z2aY_Dh@(18-vdfvn;D?sv<>UgL{Ti08$1Q+ zZI3q}yMA^LK=d?YVg({|v?d1|R?5 zL0S3fw)BZazRNNX|7P4rh7!+3tCG~O8l+m?H} z(CB>8(9LtKYIu3ohJ-9ecgk+L&!FX~Wuim&;v$>M4 zUfvn<=Eok(63Ubc>mZrd8d7(>8bG>J?PtOHih_xRYFu1Hg{t;%+hXu2#x%a%qzcab zv$X!ccoj)exoOnaco_jbGw7KryOtuf(SaR-VJ0nAe(1*AA}#QV1lMhGtzD>RoUZ;WA?~!K{8%chYn?ttlz17UpDLlhTkGcVfHY6R<2r4E{mU zq-}D?+*2gAkQYAKrk*rB%4WFC-B!eZZLg4(tR#@kUQHIzEqV48$9=Q(~J_0 zy1%LSCbkoOhRO!J+Oh#;bGuXe;~(bIE*!J@i<%_IcB7wjhB5iF#jBn5+u~fEECN2* z!QFh!m<(>%49H12Y33+?$JxKV3xW{xSs=gxkxW-@Xds^|O1`AmorDKrE8N2-@ospk z=Au%h=f!`_X|G^A;XWL}-_L@D6A~*4Yf!5RTTm$!t8y&fp5_oqvBjW{FufS`!)5m% z2g(=9Ap6Y2y(9OYOWuUVGp-K=6kqQ)kM0P^TQT{X{V$*sN$wbFb-DaUuJF*!?EJPl zJev!UsOB^UHZ2KppYTELh+kqDw+5dPFv&&;;C~=u$Mt+Ywga!8YkL2~@g67}3wAQP zrx^RaXb1(c7vwU8a2se75X(cX^$M{FH4AHS7d2}heqqg4F0!1|Na>UtAdT%3JnS!B)&zelTEj$^b0>Oyfw=P-y-Wd^#dEFRUN*C{!`aJIHi<_YA2?piC%^ zj!p}+ZnBrM?ErAM+D97B*7L8U$K zo(IR-&LF(85p+fuct9~VTSdRjs`d-m|6G;&PoWvC&s8z`TotPSoksp;RsL4VL@CHf z_3|Tn%`ObgRhLmr60<;ya-5wbh&t z#ycN_)3P_KZN5CRyG%LRO4`Ot)3vY#dNX9!f!`_>1%4Q`81E*2BRg~A-VcN7pcX#j zrbl@7`V%n z6J53(m?KRzKb)v?iCuYWbH*l6M77dY4keS!%>}*8n!@ROE4!|7mQ+YS4dff1JJC(t z6Fnuf^=dajqHpH1=|pb(po9Fr8it^;2dEk|Ro=$fxqK$^Yix{G($0m-{RCFQJ~LqUnO7jJcjr zl*N*!6WU;wtF=dLCWzD6kW;y)LEo=4wSXQDIcq5WttgE#%@*m><@H;~Q&GniA-$in z`sjWFLgychS1kIJmPtd-w6%iKkj&dGhtB%0)pyy0M<4HZ@ZY0PWLAd7FCrj&i|NRh?>hZj*&FYnyu%Ur`JdiTu&+n z78d3n)Rl6q&NwVj_jcr#s5G^d?VtV8bkkYco5lV0LiT+t8}98LW>d)|v|V3++zLbHC(NC@X#Hx?21J0M*gP2V`Yd^DYvVIr{C zSc4V)hZKf|OMSm%FVqSRC!phWSyuUAu%0fredf#TDR$|hMZihJ__F!)Nkh6z)d=NC z3q4V*K3JTetxCPgB2_)rhOSWhuXzu+%&>}*ARxUaDeRy{$xK(AC0I=9%X7dmc6?lZNqe-iM(`?Xn3x2Ov>sej6YVQJ9Q42>?4lil?X zew-S>tm{=@QC-zLtg*nh5mQojYnvVzf3!4TpXPuobW_*xYJs;9AokrXcs!Ay z;HK>#;G$*TPN2M!WxdH>oDY6k4A6S>BM0Nimf#LfboKxJXVBC=RBuO&g-=+@O-#0m zh*aPG16zY^tzQLNAF7L(IpGPa+mDsCeAK3k=IL6^LcE8l0o&)k@?dz!79yxUquQIe($zm5DG z5RdXTv)AjHaOPv6z%99mPsa#8OD@9=URvHoJ1hYnV2bG*2XYBgB!-GEoP&8fLmWGg z9NG^xl5D&3L^io&3iYweV*qhc=m+r7C#Jppo$Ygg;jO2yaFU8+F*RmPL` zYxfGKla_--I}YUT353k}nF1zt2NO?+kofR8Efl$Bb^&llgq+HV_UYJUH7M5IoN0sT z4;wDA0gs55ZI|FmJ0}^Pc}{Ji-|#jdR$`!s)Di4^g3b_Qr<*Qu2rz}R6!B^;`Lj3sKWzjMYjexX)-;f5Y+HfkctE{PstO-BZan0zdXPQ=V8 zS8cBhnQyy4oN?J~oK0zl!#S|v6h-nx5to7WkdEk0HKBm;?kcNO*A+u=%f~l&aY*+J z>%^Dz`EQ6!+SEX$>?d(~|MNWU-}JTrk}&`IR|Ske(G^iMdk04)Cxd@}{1=P0U*%L5 zMFH_$R+HUGGv|ju2Z>5x(-aIbVJLcH1S+(E#MNe9g;VZX{5f%_|Kv7|UY-CM(>vf= z!4m?QS+AL+rUyfGJ;~uJGp4{WhOOc%2ybVP68@QTwI(8kDuYf?#^xv zBmOHCZU8O(x)=GVFn%tg@TVW1)qJJ_bU}4e7i>&V?r zh-03>d3DFj&@}6t1y3*yOzllYQ++BO-q!)zsk`D(z||)y&}o%sZ-tUF>0KsiYKFg6 zTONq)P+uL5Vm0w{D5Gms^>H1qa&Z##*X31=58*r%Z@Ko=IMXX{;aiMUp-!$As3{sq z0EEk02MOsgGm7$}E%H1ys2$yftNbB%1rdo@?6~0!a8Ym*1f;jIgfcYEF(I_^+;Xdr z2a>&oc^dF3pm(UNpazXgVzuF<2|zdPGjrNUKpdb$HOgNp*V56XqH`~$c~oSiqx;8_ zEz3fHoU*aJUbFJ&?W)sZB3qOSS;OIZ=n-*#q{?PCXi?Mq4aY@=XvlNQdA;yVC0Vy+ z{Zk6OO!lMYWd`T#bS8FV(`%flEA9El;~WjZKU1YmZpG#49`ku`oV{Bdtvzyz3{k&7 zlG>ik>eL1P93F zd&!aXluU_qV1~sBQf$F%sM4kTfGx5MxO0zJy<#5Z&qzNfull=k1_CZivd-WAuIQf> zBT3&WR|VD|=nKelnp3Q@A~^d_jN3@$x2$f@E~e<$dk$L@06Paw$);l*ewndzL~LuU zq`>vfKb*+=uw`}NsM}~oY}gW%XFwy&A>bi{7s>@(cu4NM;!%ieP$8r6&6jfoq756W z$Y<`J*d7nK4`6t`sZ;l%Oen|+pk|Ry2`p9lri5VD!Gq`U#Ms}pgX3ylAFr8(?1#&dxrtJgB>VqrlWZf61(r`&zMXsV~l{UGjI7R@*NiMJLUoK*kY&gY9kC@^}Fj* zd^l6_t}%Ku<0PY71%zQL`@}L}48M!@=r)Q^Ie5AWhv%#l+Rhu6fRpvv$28TH;N7Cl z%I^4ffBqx@Pxpq|rTJV)$CnxUPOIn`u278s9#ukn>PL25VMv2mff)-RXV&r`Dwid7}TEZxXX1q(h{R6v6X z&x{S_tW%f)BHc!jHNbnrDRjGB@cam{i#zZK*_*xlW@-R3VDmp)<$}S%t*@VmYX;1h zFWmpXt@1xJlc15Yjs2&e%)d`fimRfi?+fS^BoTcrsew%e@T^}wyVv6NGDyMGHSKIQ zC>qFr4GY?#S#pq!%IM_AOf`#}tPoMn7JP8dHXm(v3UTq!aOfEXNRtEJ^4ED@jx%le zvUoUs-d|2(zBsrN0wE(Pj^g5wx{1YPg9FL1)V1JupsVaXNzq4fX+R!oVX+q3tG?L= z>=s38J_!$eSzy0m?om6Wv|ZCbYVHDH*J1_Ndajoh&?L7h&(CVii&rmLu+FcI;1qd_ zHDb3Vk=(`WV?Uq;<0NccEh0s`mBXcEtmwt6oN99RQt7MNER3`{snV$qBTp={Hn!zz z1gkYi#^;P8s!tQl(Y>|lvz{5$uiXsitTD^1YgCp+1%IMIRLiSP`sJru0oY-p!FPbI)!6{XM%)(_Dolh1;$HlghB-&e><;zU&pc=ujpa-(+S&Jj zX1n4T#DJDuG7NP;F5TkoG#qjjZ8NdXxF0l58RK?XO7?faM5*Z17stidTP|a%_N z^e$D?@~q#Pf+708cLSWCK|toT1YSHfXVIs9Dnh5R(}(I;7KhKB7RD>f%;H2X?Z9eR z{lUMuO~ffT!^ew= z7u13>STI4tZpCQ?yb9;tSM-(EGb?iW$a1eBy4-PVejgMXFIV_Ha^XB|F}zK_gzdhM z!)($XfrFHPf&uyFQf$EpcAfk83}91Y`JFJOiQ;v5ca?)a!IxOi36tGkPk4S6EW~eq z>WiK`Vu3D1DaZ}515nl6>;3#xo{GQp1(=uTXl1~ z4gdWxr-8a$L*_G^UVd&bqW_nzMM&SlNW$8|$lAfo@zb+P>2q?=+T^qNwblP*RsN?N zdZE%^Zs;yAwero1qaoqMp~|KL=&npffh981>2om!fseU(CtJ=bW7c6l{U5(07*e0~ zJRbid6?&psp)ilmYYR3ZIg;t;6?*>hoZ3uq7dvyyq-yq$zH$yyImjfhpQb@WKENSP zl;KPCE+KXzU5!)mu12~;2trrLfs&nlEVOndh9&!SAOdeYd}ugwpE-9OF|yQs(w@C9 zoXVX`LP~V>%$<(%~tE*bsq(EFm zU5z{H@Fs^>nm%m%wZs*hRl=KD%4W3|(@j!nJr{Mmkl`e_uR9fZ-E{JY7#s6i()WXB0g-b`R{2r@K{2h3T+a>82>722+$RM*?W5;Bmo6$X3+Ieg9&^TU(*F$Q3 zT572!;vJeBr-)x?cP;^w1zoAM`nWYVz^<6N>SkgG3s4MrNtzQO|A?odKurb6DGZffo>DP_)S0$#gGQ_vw@a9JDXs2}hV&c>$ zUT0;1@cY5kozKOcbN6)n5v)l#>nLFL_x?2NQgurQH(KH@gGe>F|$&@ zq@2A!EXcIsDdzf@cWqElI5~t z4cL9gg7{%~4@`ANXnVAi=JvSsj95-7V& zME3o-%9~2?cvlH#twW~99=-$C=+b5^Yv}Zh4;Mg-!LS zw>gqc=}CzS9>v5C?#re>JsRY!w|Mtv#%O3%Ydn=S9cQarqkZwaM4z(gL~1&oJZ;t; zA5+g3O6itCsu93!G1J_J%Icku>b3O6qBW$1Ej_oUWc@MI)| zQ~eyS-EAAnVZp}CQnvG0N>Kc$h^1DRJkE7xZqJ0>p<>9*apXgBMI-v87E0+PeJ-K& z#(8>P_W^h_kBkI;&e_{~!M+TXt@z8Po*!L^8XBn{of)knd-xp{heZh~@EunB2W)gd zAVTw6ZZasTi>((qpBFh(r4)k zz&@Mc@ZcI-4d639AfcOgHOU+YtpZ)rC%Bc5gw5o~+E-i+bMm(A6!uE>=>1M;V!Wl4 z<#~muol$FsY_qQC{JDc8b=$l6Y_@_!$av^08`czSm!Xan{l$@GO-zPq1s>WF)G=wv zDD8j~Ht1pFj)*-b7h>W)@O&m&VyYci&}K|0_Z*w`L>1jnGfCf@6p}Ef*?wdficVe_ zmPRUZ(C+YJU+hIj@_#IiM7+$4kH#VS5tM!Ksz01siPc-WUe9Y3|pb4u2qnn zRavJiRpa zq?tr&YV?yKt<@-kAFl3s&Kq#jag$hN+Y%%kX_ytvpCsElgFoN3SsZLC>0f|m#&Jhu zp7c1dV$55$+k78FI2q!FT}r|}cIV;zp~#6X2&}22$t6cHx_95FL~T~1XW21VFuatb zpM@6w>c^SJ>Pq6{L&f9()uy)TAWf;6LyHH3BUiJ8A4}od)9sriz~e7}l7Vr0e%(=>KG1Jay zW0azuWC`(|B?<6;R)2}aU`r@mt_#W2VrO{LcX$Hg9f4H#XpOsAOX02x^w9+xnLVAt z^~hv2guE-DElBG+`+`>PwXn5kuP_ZiOO3QuwoEr)ky;o$n7hFoh}Aq0@Ar<8`H!n} zspCC^EB=6>$q*gf&M2wj@zzfBl(w_@0;h^*fC#PW9!-kT-dt*e7^)OIU{Uw%U4d#g zL&o>6`hKQUps|G4F_5AuFU4wI)(%9(av7-u40(IaI|%ir@~w9-rLs&efOR@oQy)}{ z&T#Qf`!|52W0d+>G!h~5A}7VJky`C3^fkJzt3|M&xW~x-8rSi-uz=qBsgODqbl(W#f{Ew#ui(K)(Hr&xqZs` zfrK^2)tF#|U=K|_U@|r=M_Hb;qj1GJG=O=d`~#AFAccecIaq3U`(Ds1*f*TIs=IGL zp_vlaRUtFNK8(k;JEu&|i_m39c(HblQkF8g#l|?hPaUzH2kAAF1>>Yykva0;U@&oRV8w?5yEK??A0SBgh?@Pd zJg{O~4xURt7!a;$rz9%IMHQeEZHR8KgFQixarg+MfmM_OeX#~#&?mx44qe!wt`~dd zqyt^~ML>V>2Do$huU<7}EF2wy9^kJJSm6HoAD*sRz%a|aJWz_n6?bz99h)jNMp}3k ztPVbos1$lC1nX_OK0~h>=F&v^IfgBF{#BIi&HTL}O7H-t4+wwa)kf3AE2-Dx@#mTA z!0f`>vz+d3AF$NH_-JqkuK1C+5>yns0G;r5ApsU|a-w9^j4c+FS{#+7- zH%skr+TJ~W_8CK_j$T1b;$ql_+;q6W|D^BNK*A+W5XQBbJy|)(IDA=L9d>t1`KX2b zOX(Ffv*m?e>! zS3lc>XC@IqPf1g-%^4XyGl*1v0NWnwZTW?z4Y6sncXkaA{?NYna3(n@(+n+#sYm}A zGQS;*Li$4R(Ff{obl3#6pUsA0fKuWurQo$mWXMNPV5K66V!XYOyc})^>889Hg3I<{V^Lj9($B4Zu$xRr=89-lDz9x`+I8q(vEAimx1K{sTbs|5x7S zZ+7o$;9&9>@3K;5-DVzGw=kp7ez%1*kxhGytdLS>Q)=xUWv3k_x(IsS8we39Tijvr z`GKk>gkZTHSht;5q%fh9z?vk%sWO}KR04G9^jleJ^@ovWrob7{1xy7V=;S~dDVt%S za$Q#Th%6g1(hiP>hDe}7lcuI94K-2~Q0R3A1nsb7Y*Z!DtQ(Ic<0;TDKvc6%1kBdJ z$hF!{uALB0pa?B^TC}#N5gZ|CKjy|BnT$7eaKj;f>Alqdb_FA3yjZ4CCvm)D&ibL) zZRi91HC!TIAUl<|`rK_6avGh`!)TKk=j|8*W|!vb9>HLv^E%t$`@r@piI(6V8pqDG zBON7~=cf1ZWF6jc{qkKm;oYBtUpIdau6s+<-o^5qNi-p%L%xAtn9OktFd{@EjVAT% z#?-MJ5}Q9QiK_jYYWs+;I4&!N^(mb!%4zx7qO6oCEDn=8oL6#*9XIJ&iJ30O`0vsFy|fEVkw}*jd&B6!IYi+~Y)qv6QlM&V9g0 zh)@^BVDB|P&#X{31>G*nAT}Mz-j~zd>L{v{9AxrxKFw8j;ccQ$NE0PZCc(7fEt1xd z`(oR2!gX6}R+Z77VkDz^{I)@%&HQT5q+1xlf*3R^U8q%;IT8-B53&}dNA7GW`Ki&= z$lrdH zDCu;j$GxW<&v_4Te7=AE2J0u1NM_7Hl9$u{z(8#%8vvrx2P#R7AwnY|?#LbWmROa; zOJzU_*^+n(+k;Jd{e~So9>OF>fPx$Hb$?~K1ul2xr>>o@**n^6IMu8+o3rDp(X$cC z`wQt9qIS>yjA$K~bg{M%kJ00A)U4L+#*@$8UlS#lN3YA{R{7{-zu#n1>0@(#^eb_% zY|q}2)jOEM8t~9p$X5fpT7BZQ1bND#^Uyaa{mNcFWL|MoYb@>y`d{VwmsF&haoJuS2W7azZU0{tu#Jj_-^QRc35tjW~ae&zhKk!wD}#xR1WHu z_7Fys#bp&R?VXy$WYa$~!dMxt2@*(>@xS}5f-@6eoT%rwH zv_6}M?+piNE;BqaKzm1kK@?fTy$4k5cqYdN8x-<(o6KelwvkTqC3VW5HEnr+WGQlF zs`lcYEm=HPpmM4;Ich7A3a5Mb3YyQs7(Tuz-k4O0*-YGvl+2&V(B&L1F8qfR0@vQM-rF<2h-l9T12eL}3LnNAVyY_z51xVr$%@VQ-lS~wf3mnHc zoM({3Z<3+PpTFCRn_Y6cbxu9v>_>eTN0>hHPl_NQQuaK^Mhrv zX{q#80ot;ptt3#js3>kD&uNs{G0mQp>jyc0GG?=9wb33hm z`y2jL=J)T1JD7eX3xa4h$bG}2ev=?7f>-JmCj6){Upo&$k{2WA=%f;KB;X5e;JF3IjQBa4e-Gp~xv- z|In&Rad7LjJVz*q*+splCj|{7=kvQLw0F@$vPuw4m^z=B^7=A4asK_`%lEf_oIJ-O z{L)zi4bd#&g0w{p1$#I&@bz3QXu%Y)j46HAJKWVfRRB*oXo4lIy7BcVl4hRs<%&iQ zr|)Z^LUJ>qn>{6y`JdabfNNFPX7#3`x|uw+z@h<`x{J4&NlDjnknMf(VW_nKWT!Jh zo1iWBqT6^BR-{T=4Ybe+?6zxP_;A5Uo{}Xel%*=|zRGm1)pR43K39SZ=%{MDCS2d$~}PE-xPw4ZK6)H;Zc&0D5p!vjCn0wCe&rVIhchR9ql!p2`g0b@JsC^J#n_r*4lZ~u0UHKwo(HaHUJDHf^gdJhTdTW z3i7Zp_`xyKC&AI^#~JMVZj^9WsW}UR#nc#o+ifY<4`M+?Y9NTBT~p`ONtAFf8(ltr*ER-Ig!yRs2xke#NN zkyFcaQKYv>L8mQdrL+#rjgVY>Z2_$bIUz(kaqL}cYENh-2S6BQK-a(VNDa_UewSW` zMgHi<3`f!eHsyL6*^e^W7#l?V|42CfAjsgyiJsA`yNfAMB*lAsJj^K3EcCzm1KT zDU2+A5~X%ax-JJ@&7>m`T;;}(-e%gcYQtj}?ic<*gkv)X2-QJI5I0tA2`*zZRX(;6 zJ0dYfMbQ+{9Rn3T@Iu4+imx3Y%bcf2{uT4j-msZ~eO)5Z_T7NC|Nr3)|NWjomhv=E zXaVin)MY)`1QtDyO7mUCjG{5+o1jD_anyKn73uflH*ASA8rm+S=gIfgJ);>Zx*hNG z!)8DDCNOrbR#9M7Ud_1kf6BP)x^p(|_VWCJ+(WGDbYmnMLWc?O4zz#eiP3{NfP1UV z(n3vc-axE&vko^f+4nkF=XK-mnHHQ7>w05$Q}iv(kJc4O3TEvuIDM<=U9@`~WdKN* zp4e4R1ncR_kghW}>aE$@OOc~*aH5OOwB5U*Z)%{LRlhtHuigxH8KuDwvq5{3Zg{Vr zrd@)KPwVKFP2{rXho(>MTZZfkr$*alm_lltPob4N4MmhEkv`J(9NZFzA>q0Ch;!Ut zi@jS_=0%HAlN+$-IZGPi_6$)ap>Z{XQGt&@ZaJ(es!Po5*3}>R4x66WZNsjE4BVgn z>}xm=V?F#tx#e+pimNPH?Md5hV7>0pAg$K!?mpt@pXg6UW9c?gvzlNe0 z3QtIWmw$0raJkjQcbv-7Ri&eX6Ks@@EZ&53N|g7HU<;V1pkc&$3D#8k!coJ=^{=vf z-pCP;vr2#A+i#6VA?!hs6A4P@mN62XYY$#W9;MwNia~89i`=1GoFESI+%Mbrmwg*0 zbBq4^bA^XT#1MAOum)L&ARDXJ6S#G>&*72f50M1r5JAnM1p7GFIv$Kf9eVR(u$KLt z9&hQ{t^i16zL1c(tRa~?qr?lbSN;1k;%;p*#gw_BwHJRjcYPTj6>y-rw*dFTnEs95 z`%-AoPL!P16{=#RI0 zUb6#`KR|v^?6uNnY`zglZ#Wd|{*rZ(x&Hk8N6ob6mpX~e^qu5kxvh$2TLJA$M=rx zc!#ot+sS+-!O<0KR6+Lx&~zgEhCsbFY{i_DQCihspM?e z-V}HemMAvFzXR#fV~a=Xf-;tJ1edd}Mry@^=9BxON;dYr8vDEK<<{ zW~rg(ZspxuC&aJo$GTM!9_sXu(EaQJNkV9AC(ob#uA=b4*!Uf}B*@TK=*dBvKKPAF z%14J$S)s-ws9~qKsf>DseEW(ssVQ9__YNg}r9GGx3AJiZR@w_QBlGP>yYh0lQCBtf zx+G;mP+cMAg&b^7J!`SiBwC81M_r0X9kAr2y$0(Lf1gZK#>i!cbww(hn$;fLIxRf? z!AtkSZc-h76KGSGz%48Oe`8ZBHkSXeVb!TJt_VC>$m<#}(Z}!(3h631ltKb3CDMw^fTRy%Ia!b&at`^g7Ew-%WLT9(#V0OP9CE?uj62s>`GI3NA z!`$U+i<`;IQyNBkou4|-7^9^ylac-Xu!M+V5p5l0Ve?J0wTSV+$gYtoc=+Ve*OJUJ z$+uIGALW?}+M!J9+M&#bT=Hz@{R2o>NtNGu1yS({pyteyb>*sg4N`KAD?`u3F#C1y z2K4FKOAPASGZTep54PqyCG(h3?kqQQAxDSW@>T2d!n;9C8NGS;3A8YMRcL>b=<<%M zMiWf$jY;`Ojq5S{kA!?28o)v$;)5bTL<4eM-_^h4)F#eeC2Dj*S`$jl^yn#NjJOYT zx%yC5Ww@eX*zsM)P(5#wRd=0+3~&3pdIH7CxF_2iZSw@>kCyd z%M}$1p((Bidw4XNtk&`BTkU{-PG)SXIZ)yQ!Iol6u8l*SQ1^%zC72FP zLvG>_Z0SReMvB%)1@+et0S{<3hV@^SY3V~5IY(KUtTR{*^xJ^2NN{sIMD9Mr9$~(C$GLNlSpzS=fsbw-DtHb_T|{s z9OR|sx!{?F``H!gVUltY7l~dx^a(2;OUV^)7 z%@hg`8+r&xIxmzZ;Q&v0X%9P)U0SE@r@(lKP%TO(>6I_iF{?PX(bez6v8Gp!W_nd5 z<8)`1jcT)ImNZp-9rr4_1MQ|!?#8sJQx{`~7)QZ75I=DPAFD9Mt{zqFrcrXCU9MG8 zEuGcy;nZ?J#M3!3DWW?Zqv~dnN6ijlIjPfJx(#S0cs;Z=jDjKY|$w2s4*Xa1Iz953sN2Lt!Vmk|%ZwOOqj`sA--5Hiaq8!C%LV zvWZ=bxeRV(&%BffMJ_F~~*FdcjhRVNUXu)MS(S#67rDe%Ler=GS+WysC1I2=Bmbh3s6wdS}o$0 zz%H08#SPFY9JPdL6blGD$D-AaYi;X!#zqib`(XX*i<*eh+2UEPzU4}V4RlC3{<>-~ zadGA8lSm>b7Z!q;D_f9DT4i)Q_}ByElGl*Cy~zX%IzHp)@g-itZB6xM70psn z;AY8II99e6P2drgtTG5>`^|7qg`9MTp%T~|1N3tBqV}2zgow3TFAH{XPor0%=HrkXnKyxyozHlJ6 zd3}OWkl?H$l#yZqOzZbMI+lDLoH48;s10!m1!K87g;t}^+A3f3e&w{EYhVPR0Km*- zh5-ku$Z|Ss{2?4pGm(Rz!0OQb^_*N`)rW{z)^Cw_`a(_L9j=&HEJl(!4rQy1IS)>- zeTIr>hOii`gc(fgYF(cs$R8l@q{mJzpoB5`5r>|sG zBpsY}RkY(g5`bj~D>(;F8v*DyjX(#nVLSs>)XneWI&%Wo>a0u#4A?N<1SK4D}&V1oN)76 z%S>a2n3n>G`YY1>0Hvn&AMtMuI_?`5?4y3w2Hnq4Qa2YH5 zxKdfM;k467djL31Y$0kd9FCPbU=pHBp@zaIi`Xkd80;%&66zvSqsq6%aY)jZacfvw ztkWE{ZV6V2WL9e}Dvz|!d96KqVkJU@5ryp#rReeWu>mSrOJxY^tWC9wd0)$+lZc%{ zY=c4#%OSyQJvQUuy^u}s8DN8|8T%TajOuaY^)R-&8s@r9D`(Ic4NmEu)fg1f!u`xUb;9t#rM z>}cY=648@d5(9A;J)d{a^*ORdVtJrZ77!g~^lZ9@)|-ojvW#>)Jhe8$7W3mhmQh@S zU=CSO+1gSsQ+Tv=x-BD}*py_Ox@;%#hPb&tqXqyUW9jV+fonnuCyVw=?HR>dAB~Fg z^vl*~y*4|)WUW*9RC%~O1gHW~*tJb^a-j;ae2LRNo|0S2`RX>MYqGKB^_ng7YRc@! zFxg1X!VsvXkNuv^3mI`F2=x6$(pZdw=jfYt1ja3FY7a41T07FPdCqFhU6%o|Yb6Z4 zpBGa=(ao3vvhUv#*S{li|EyujXQPUV;0sa5!0Ut)>tPWyC9e0_9(=v*z`TV5OUCcx zT=w=^8#5u~7<}8Mepqln4lDv*-~g^VoV{(+*4w(q{At6d^E-Usa2`JXty++Oh~on^ z;;WHkJsk2jvh#N|?(2PLl+g!M0#z_A;(#Uy=TzL&{Ei5G9#V{JbhKV$Qmkm%5tn!CMA? z@hM=b@2DZWTQ6>&F6WCq6;~~WALiS#@{|I+ucCmD6|tBf&e;$_)%JL8$oIQ%!|Xih1v4A$=7xNO zZVz$G8;G5)rxyD+M0$20L$4yukA_D+)xmK3DMTH3Q+$N&L%qB)XwYx&s1gkh=%qGCCPwnwhbT4p%*3R)I}S#w7HK3W^E%4w z2+7ctHPx3Q97MFYB48HfD!xKKb(U^K_4)Bz(5dvwyl*R?)k;uHEYVi|{^rvh)w7}t z`tnH{v9nlVHj2ign|1an_wz0vO)*`3RaJc#;(W-Q6!P&>+@#fptCgtUSn4!@b7tW0&pE2Qj@7}f#ugu4*C)8_}AMRuz^WG zc)XDcOPQjRaGptRD^57B83B-2NKRo!j6TBAJntJPHNQG;^Oz}zt5F^kId~miK3J@l ztc-IKp6qL!?u~q?qfGP0I~$5gvq#-0;R(oLU@sYayr*QH95fnrYA*E|n%&FP@Cz`a zSdJ~(c@O^>qaO`m9IQ8sd8!L<+)GPJDrL7{4{ko2gWOZel^3!($Gjt|B&$4dtfTmBmC>V`R&&6$wpgvdmns zxcmfS%9_ZoN>F~azvLFtA(9Q5HYT#A(byGkESnt{$Tu<73$W~reB4&KF^JBsoqJ6b zS?$D7DoUgzLO-?P`V?5_ub$nf1p0mF?I)StvPomT{uYjy!w&z$t~j&en=F~hw|O(1 zlV9$arQmKTc$L)Kupwz_zA~deT+-0WX6NzFPh&d+ly*3$%#?Ca9Z9lOJsGVoQ&1HNg+)tJ_sw)%oo*DK)iU~n zvL``LqTe=r=7SwZ@LB)9|3QB5`0(B9r(iR}0nUwJss-v=dXnwMRQFYSRK1blS#^g(3@z{`=8_CGDm!LESTWig zzm1{?AG&7`uYJ;PoFO$o8RWuYsV26V{>D-iYTnvq7igWx9@w$EC*FV^vpvDl@i9yp zPIqiX@hEZF4VqzI3Y)CHhR`xKN8poL&~ak|wgbE4zR%Dm(a@?bw%(7(!^>CM!^4@J z6Z)KhoQP;WBq_Z_&<@i2t2&xq>N>b;Np2rX?yK|-!14iE2T}E|jC+=wYe~`y38g3J z8QGZquvqBaG!vw&VtdXWX5*i5*% zJP~7h{?&E|<#l{klGPaun`IgAJ4;RlbRqgJz5rmHF>MtJHbfqyyZi53?Lhj=(Ku#& z__ubmZIxzSq3F90Xur!1)Vqe6b@!ueHA!93H~jdHmaS5Q^CULso}^poy)0Op6!{^9 zWyCyyIrdBP4fkliZ%*g+J-A!6VFSRF6Liu6G^^=W>cn81>4&7(c7(6vCGSAJ zQZ|S3mb|^Wf=yJ(h~rq`iiW~|n#$+KcblIR<@|lDtm!&NBzSG-1;7#YaU+-@=xIm4 zE}edTYd~e&_%+`dIqqgFntL-FxL3!m4yTNt<(^Vt9c6F(`?9`u>$oNxoKB29<}9FE zgf)VK!*F}nW?}l95%RRk8N4^Rf8)Xf;drT4<|lUDLPj^NPMrBPL;MX&0oGCsS za3}vWcF(IPx&W6{s%zwX{UxHX2&xLGfT{d9bWP!g;Lg#etpuno$}tHoG<4Kd*=kpU z;4%y(<^yj(UlG%l-7E9z_Kh2KoQ19qT3CR@Ghr>BAgr3Vniz3LmpC4g=g|A3968yD2KD$P7v$ zx9Q8`2&qH3&y-iv0#0+jur@}k`6C%7fKbCr|tHX2&O%r?rBpg`YNy~2m+ z*L7dP$RANzVUsG_Lb>=__``6vA*xpUecuGsL+AW?BeSwyoQfDlXe8R1*R1M{0#M?M zF+m19`3<`gM{+GpgW^=UmuK*yMh3}x)7P738wL8r@(Na6%ULPgbPVTa6gh5Q(SR0f znr6kdRpe^(LVM;6Rt(Z@Lsz3EX*ry6(WZ?w>#ZRelx)N%sE+MN>5G|Z8{%@b&D+Ov zPU{shc9}%;G7l;qbonIb_1m^Qc8ez}gTC-k02G8Rl?7={9zBz8uRX2{XJQ{vZhs67avlRn| zgRtWl0Lhjet&!YC47GIm%1gdq%T24_^@!W3pCywc89X4I5pnBCZDn(%!$lOGvS*`0!AoMtqxNPFgaMR zwoW$p;8l6v%a)vaNsesED3f}$%(>zICnoE|5JwP&+0XI}JxPccd+D^gx`g`=GsUc0 z9Uad|C+_@_0%JmcObGnS@3+J^0P!tg+fUZ_w#4rk#TlJYPXJiO>SBxzs9(J;XV9d{ zmTQE1(K8EYaz9p^XLbdWudyIPJlGPo0U*)fAh-jnbfm@SYD_2+?|DJ-^P+ojG{2{6 z>HJtedEjO@j_tqZ4;Zq1t5*5cWm~W?HGP!@_f6m#btM@46cEMhhK{(yI&jG)fwL1W z^n_?o@G8a-jYt!}$H*;{0#z8lANlo!9b@!c5K8<(#lPlpE!z86Yq#>WT&2} z;;G1$pD%iNoj#Z=&kij5&V1KHIhN-h<;{HC5wD)PvkF>CzlQOEx_0;-TJ*!#&{Wzt zKcvq^SZIdop}y~iouNqtU7K7+?eIz-v_rfNM>t#i+dD$s_`M;sjGubTdP)WI*uL@xPOLHt#~T<@Yz>xt50ZoTw;a(a}lNiDN-J${gOdE zx?8LOA|tv{Mb}=TTR=LcqMqbCJkKj+@;4Mu)Cu0{`~ohix6E$g&tff)aHeUAQQ%M? zIN4uSUTzC1iMEWL*W-in1y)C`E+R8j?4_?X4&2Zv5?QdkNMz(k} zw##^Ikx`#_s>i&CO_mu@vJJ*|3ePRDl5pq$9V^>D;g0R%l>lw;ttyM6Sy`NBF{)Lr zSk)V>mZr96+aHY%vTLLt%vO-+juw6^SO_ zYGJaGeWX6W(TOQx=5oTGXOFqMMU*uZyt>MR-Y`vxW#^&)H zk0!F8f*@v6NO@Z*@Qo)+hlX40EWcj~j9dGrLaq%1;DE_%#lffXCcJ;!ZyyyZTz74Q zb2WSly6sX{`gQeToQsi1-()5EJ1nJ*kXGD`xpXr~?F#V^sxE3qSOwRSaC9x9oa~jJ zTG9`E|q zC5Qs1xh}jzb5UPYF`3N9YuMnI7xsZ41P;?@c|%w zl=OxLr6sMGR+`LStLvh)g?fA5p|xbUD;yFAMQg&!PEDYxVYDfA>oTY;CFt`cg?Li1 z0b})!9Rvw&j#*&+D2))kXLL z0+j=?7?#~_}N-qdEIP>DQaZh#F(#e0WNLzwUAj@r694VJ8?Dr5_io2X49XYsG^ zREt0$HiNI~6VV!ycvao+0v7uT$_ilKCvsC+VDNg7yG1X+eNe^3D^S==F3ByiW0T^F zH6EsH^}Uj^VPIE&m)xlmOScYR(w750>hclqH~~dM2+;%GDXT`u4zG!p((*`Hwx41M z4KB+`hfT(YA%W)Ve(n+Gu9kuXWKzxg{1ff^xNQw>w%L-)RySTk9kAS92(X0Shg^Q? zx1YXg_TLC^?h6!4mBqZ9pKhXByu|u~gF%`%`vdoaGBN3^j4l!4x?Bw4Jd)Z4^di}! zXlG1;hFvc>H?bmmu1E7Vx=%vahd!P1#ZGJOJYNbaek^$DHt`EOE|Hlij+hX>ocQFSLVu|wz`|KVl@Oa;m2k6b*mNK2Vo{~l9>Qa3@B7G7#k?)aLx;w6U ze8bBq%vF?5v>#TspEoaII!N}sRT~>bh-VWJ7Q*1qsz%|G)CFmnttbq$Ogb{~YK_=! z{{0vhlW@g!$>|}$&4E3@k`KPElW6x#tSX&dfle>o!irek$NAbDzdd2pVeNzk4&qgJ zXvNF0$R96~g0x+R1igR=Xu&X_Hc5;!Ze&C)eUTB$9wW&?$&o8Yxhm5s(S`;?{> z*F?9Gr0|!OiKA>Rq-ae=_okB6&yMR?!JDer{@iQgIn=cGxs-u^!8Q$+N&pfg2WM&Z zulHu=Uh~U>fS{=Nm0x>ACvG*4R`Dx^kJ65&Vvfj`rSCV$5>c04N26Rt2S?*kh3JKq z9(3}5T?*x*AP(X2Ukftym0XOvg~r6Ms$2x&R&#}Sz23aMGU&7sU-cFvE3Eq`NBJe84VoftWF#v7PDAp`@V zRFCS24_k~;@~R*L)eCx@Q9EYmM)Sn}HLbVMyxx%{XnMBDc-YZ<(DXDBYUt8$u5Zh} zBK~=M9cG$?_m_M61YG+#|9Vef7LfbH>(C21&aC)x$^Lg}fa#SF){RX|?-xZjSOrn# z2ZAwUF)$VB<&S;R3FhNSQOV~8w%A`V9dWyLiy zgt7G=Z4t|zU3!dh5|s(@XyS|waBr$>@=^Dspmem8)@L`Ns{xl%rGdX!R(BiC5C7Vo zXetb$oC_iXS}2x_Hy}T(hUUNbO47Q@+^4Q`h>(R-;OxCyW#eoOeC51jzxnM1yxBrp zz6}z`(=cngs6X05e79o_B7@3K|Qpe3n38Py_~ zpi?^rj!`pq!7PHGliC$`-8A^Ib?2qgJJCW+(&TfOnFGJ+@-<<~`7BR0f4oSINBq&R z2CM`0%WLg_Duw^1SPwj-{?BUl2Y=M4e+7yL1{C&&f&zjF06#xf>VdLozgNye(BNgSD`=fFbBy0HIosLl@JwCQl^s;eTnc( z3!r8G=K>zb`|bLLI0N|eFJk%s)B>oJ^M@AQzqR;HUjLsOqW<0v>1ksT_#24*U@R3HJu*A^#1o#P3%3_jq>icD@<`tqU6ICEgZrME(xX#?i^Z z%Id$_uyQGlFD-CcaiRtRdGn|K`Lq5L-rx7`vYYGH7I=eLfHRozPiUtSe~Tt;IN2^gCXmf2#D~g2@9bhzK}3nphhG%d?V7+Zq{I2?Gt*!NSn_r~dd$ zqkUOg{U=MI?Ehx@`(X%rQB?LP=CjJ*V!rec{#0W2WshH$X#9zep!K)tzZoge*LYd5 z@g?-j5_mtMp>_WW`p*UNUZTFN{_+#m*bJzt{hvAdkF{W40{#L3w6gzPztnsA_4?&0 z(+>pv!zB16rR-(nm(^c>Z(its{ny677vT8sF564^mlZvJ!h65}OW%Hn|2OXbOQM%b z{6C54Z2v;^hyMQ;UH+HwFD2!F!VlQ}6Z{L0_9g5~CH0@Mqz?ZC`^QkhOU#$Lx<4`B zyZsa9uPF!rZDo8ZVfzzR#raQ>5|)k~_Ef*wDqG^76o)j!C4 zykvT*o$!-MBko@?{b~*Zf2*YMlImrK`cEp|#D7f%Twm<|C|dWD \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index af55214d..f3000f35 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -142,10 +142,28 @@ function build_ancillary_repository() { # environment variables, and the script will work as expected. # -if [[ -z "$UPSTREAM_BRANCH" ]]; then - echo "UPSTREAM_BRANCH is not set." - exit 1 +# +# Set UPSTREAM_BRANCH. This will determine which version of the linux package +# mirror is used. +# +if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + echo "UPSTREAM_PRODUCT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ + "Ensure branch.config is properly formatted with e.g." \ + "UPSTREAM_BRANCH=\"\"" + exit 1 + fi + echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." +else + UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" fi +echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" AWS_S3_URI_VIRTUALIZATION=$(resolve_s3_uri \ "$AWS_S3_URI_VIRTUALIZATION" \ diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 482ccce6..3bc2fdf6 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -128,6 +128,29 @@ while ! curl --output /dev/null --silent --head --fail \ sleep 1 done +# +# Set UPSTREAM_BRANCH. This will determine which version of the linux package +# mirror is used. +# +if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + echo "UPSTREAM_PRODUCT_BRANCH is not set." + if ! source "$TOP/branch.config" 2>/dev/null; then + echo "No branch.config file found in repo root." + exit 1 + fi + + if [[ -z "$UPSTREAM_BRANCH" ]]; then + echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ + "Ensure branch.config is properly formatted with e.g." \ + "UPSTREAM_BRANCH=\"\"" + exit 1 + fi + echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." +else + UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" +fi +echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + pkg_mirror_secondary='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_SECONDARY" ]]; then pkg_mirror_secondary="$DELPHIX_PACKAGE_MIRROR_SECONDARY" From de191db1d43b9fd5d85ebb4721a8d947884d9eab Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Jan 2020 13:55:01 -0800 Subject: [PATCH 017/197] Modify propagation type of root dataset mount before mounting children (#402) (#408) Co-authored-by: John Gallagher --- .../vm-artifacts/90-raw-disk-image.binary | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 8d0e5623..3efa513e 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -138,6 +138,37 @@ zfs create \ zfs mount "rpool/ROOT/$FSNAME/root" +# +# We are later going to recursively bind mount /proc/, /sys/, and /dev/ +# beneath the root dataset. Before doing that, we need to change the root +# dataset's mount so that it has type 'slave'. If were to leave it 'shared', +# the following would happen: +# +# - The mount of $DIRECTORY/sys/ would be propagated to other existing +# namespaces. +# - We would recursively change the propagation type of $DIRECTORY/sys/ to +# be 'slave', for reasons explained below. +# - After finishing our work, we would 'umount -R $DIRECTORY/sys/'. For the +# children operated on recursively, this event would _not_ be propagated +# because they are not longer shared. +# - We would unmount $DIRECTORY. This would succeed, but propagation of it +# would silently fail, because it is busy in other namespaces, because +# $DIRECTORY/sys/ is still mounted on it. +# - We would export the pool, which would fail because the pool is busy +# because root filesystem is still mounted in other namespaces. +# +# To prevent this, we can change the propagation type of $DIRECTORY so that +# the bind mount of /sys and the others is never propagated to other +# namespaces. +# +# Also, since we are going to change the propagation type, we need to change +# it before before mounting _any_ children. Otherwise we would end up in the +# same situation, but with a different child: the mount would be proagated, +# the unmount would not, and we would end up with EBUSY errors when exporting +# the pool because some filesystems are mounted in other namespaces. +# +mount --make-slave "$DIRECTORY" + zfs create \ -o mountpoint=legacy \ "rpool/ROOT/$FSNAME/home" @@ -235,9 +266,18 @@ EOF # need to have the /dev, /proc, and /sys mountpoints present in that chroot # environment, which is why we bind mount here. # -mount --make-slave "$DIRECTORY" for dir in /dev /proc /sys; do mount --rbind "$dir" "${DIRECTORY}${dir}" + # + # Bind mounts are placed in the same peer group as the mount being + # copied. This means that when we later need to 'umount -R' this + # directory, the unmount events for any children of this mount will + # be propagated to the original mount point. So, for instance, when + # we unmount $DIRECTORY/sys/fs/cgroup, that will also attempt to + # unmount /sys/fs/cgroup. To prevent this from happening, we need to + # change the mount propagation type to prevent the unmount from being + # propagated. + # mount --make-rslave "${DIRECTORY}${dir}" done From fa01a2cdf2e8064e7181d00f7f1954a3c9e92362 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Mon, 13 Jan 2020 19:17:01 -0500 Subject: [PATCH 018/197] DLPX-68030 [Backport of Issue DLPX-67937 to 6.0.0.0] need OCI image (#410) --- gradle-lib/util.gradle | 2 +- live-build/build.gradle | 3 ++- .../config/hooks/vm-artifacts/91-qcow2-disk-image.binary | 3 ++- scripts/run-live-build.sh | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/gradle-lib/util.gradle b/gradle-lib/util.gradle index abb1b00b..feed1fba 100644 --- a/gradle-lib/util.gradle +++ b/gradle-lib/util.gradle @@ -19,7 +19,7 @@ import java.nio.file.Paths import java.nio.file.attribute.PosixFilePermissions import java.util.regex.Pattern -ext.allPlatforms = ["aws", "azure", "esx", "gcp", "hyperv", "kvm"] +ext.allPlatforms = ["aws", "azure", "esx", "gcp", "hyperv", "kvm", "oci"] ext.allVariants = new File("${rootProject.projectDir}/live-build/variants").list() ext.allInternal = allVariants.findAll { it.startsWith("internal-") } ext.allExternal = allVariants.findAll { it.startsWith("external-") } diff --git a/live-build/build.gradle b/live-build/build.gradle index 24c2480c..c9460587 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -47,7 +47,8 @@ def artifactTypes = ["aws": "vmdk", "esx": "ova", "gcp": "gcp.tar.gz", "hyperv": "vhdx", - "kvm": "qcow2"] + "kvm": "qcow2", + "oci": "qcow2"] def configDir = "configuration" diff --git a/live-build/config/hooks/vm-artifacts/91-qcow2-disk-image.binary b/live-build/config/hooks/vm-artifacts/91-qcow2-disk-image.binary index 3c82c527..6a94f8a9 100755 --- a/live-build/config/hooks/vm-artifacts/91-qcow2-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/91-qcow2-disk-image.binary @@ -21,7 +21,8 @@ # assumed to be a "raw" disk image, into a qcow2 formated disk image. # -[[ "$APPLIANCE_PLATFORM" == kvm ]] || exit 0 +[[ "$APPLIANCE_PLATFORM" == kvm ]] || [[ "$APPLIANCE_PLATFORM" == oci ]] || + exit 0 source /usr/share/livecd-rootfs/live-build/functions rm -f "$ARTIFACT_NAME.qcow2" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 3bc2fdf6..04d1ab7c 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -242,6 +242,7 @@ esx) vm_artifact_ext=ova ;; gcp) vm_artifact_ext=gcp.tar.gz ;; hyperv) vm_artifact_ext=vhdx ;; kvm) vm_artifact_ext=qcow2 ;; +oci) vm_artifact_ext=qcow2 ;; *) echo "Invalid platform" exit 1 From 28413934be8544b75bcda800152d034b62fe88a9 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 27 Jan 2020 10:48:44 -0800 Subject: [PATCH 019/197] Remove (now) unused Docker related logic (#405) (#411) --- .../appliance-build.bootstrap/tasks/main.yml | 16 ---- docker/Dockerfile | 87 ----------------- scripts/docker-run.sh | 93 ------------------- 3 files changed, 196 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100755 scripts/docker-run.sh diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 40a77d34..9b067a02 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -15,9 +15,6 @@ # --- -- shell: git rev-parse --show-toplevel 2>/dev/null - register: toplevel - - apt: update_cache: yes @@ -50,16 +47,3 @@ - modprobe: name: zfs state: present - -- apt: - name: - - docker.io - - python3-docker - state: present - -# -# We can't use the docker_image module because it doesn't yet support passing -# the 'network' parameter: https://github.com/ansible/ansible/pull/50313, which -# we need to be able to fetch things from Artifactory. -# -- shell: docker build --network host --tag "appliance-build:latest" "{{ toplevel.stdout }}/docker" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index f2c80968..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,87 +0,0 @@ -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:bionic-20180426 - -MAINTAINER Prakash Surya - -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root - -WORKDIR /root -SHELL ["/bin/bash", "-c"] - -RUN \ - apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:ansible/ansible && \ - apt-get update && \ - apt-get install -y \ - ansible \ - aptly \ - awscli \ - bc \ - coreutils \ - equivs \ - gdisk \ - git \ - java-package \ - jq \ - kpartx \ - libxt6 \ - livecd-rootfs \ - make \ - man \ - openjdk-8-jre-headless \ - pigz \ - rename \ - shellcheck \ - vim \ - zfsutils-linux && \ - rm -rf /var/lib/apt/lists/* - -# -# Download and install Gradle. If downloading from Artifactory doesn't work, -# fall back to downloading from the official Gradle site. This allows us to -# build the Docker image when not on the Delphix network, which is useful for -# running style check via TravisCI. -# -RUN \ - ( wget -nv http://artifactory.delphix.com/artifactory/gradle-distributions/gradle-5.1-bin.zip || \ - wget -nv https://services.gradle.org/distributions/gradle-5.1-bin.zip ) && \ - sha256sum -c <<< '7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d gradle-5.1-bin.zip' && \ - mkdir /opt/gradle && \ - unzip -d /opt/gradle gradle-5.1-bin.zip && \ - rm gradle-5.1-bin.zip - -RUN wget -nv -O /usr/local/bin/shfmt \ - https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_amd64 && \ - chmod +x /usr/local/bin/shfmt - -RUN \ - git clone https://github.com/willthames/ansible-lint /opt/ansible-lint && \ - cd /opt/ansible-lint && \ - git checkout v3.4.21 && \ - git branch -D master -ENV PYTHONPATH="${PYTHONPATH}:/opt/ansible-lint/lib" -ENV PATH="${PATH}:/opt/ansible-lint/bin:/opt/gradle/gradle-5.1/bin" - -# -# Set up the Gradle home directory to be located in a gitignored directory -# inside the repo. This way the cache of downloaded dependencies is preserved -# when the container running a build is destroyed. -# -ENV GRADLE_USER_HOME=/opt/appliance-build/.gradleUserHome diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh deleted file mode 100755 index 733200c4..00000000 --- a/scripts/docker-run.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOP=$(git rev-parse --show-toplevel 2>/dev/null) - -if [[ -z "$TOP" ]]; then - echo "Must be run inside the git repsitory." - exit 1 -fi - -set -o errexit -set -o xtrace - -# -# When running this script interactively (e.g. from an interactive -# terminal session) we want to use the "-ti" options. But, if we use -# these options from a Jenkins job, the command will fail because there -# won't be a TTY available. Thus, we need to check to see if a TTY is -# available before we try to use the "-ti" options. -# -if tty -s; then - DOCKER_RUN="docker run -ti" -else - DOCKER_RUN="docker run" -fi - -# -# Set UPSTREAM_BRANCH. This will determine which version of the linux package -# mirror is used. -# -if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then - echo "UPSTREAM_PRODUCT_BRANCH is not set." - if ! source "$TOP/branch.config" 2>/dev/null; then - echo "No branch.config file found in repo root." - exit 1 - fi - - if [[ -z "$UPSTREAM_BRANCH" ]]; then - echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ - "Ensure branch.config is properly formatted with e.g." \ - "UPSTREAM_BRANCH=\"\"" - exit 1 - fi - echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." -else - UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" -fi -echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" - -$DOCKER_RUN --rm \ - --privileged \ - --network host \ - --ipc "none" \ - --volume /dev:/dev \ - --env AWS_S3_PREFIX_VIRTUALIZATION \ - --env AWS_S3_PREFIX_MASKING \ - --env AWS_S3_PREFIX_USERLAND_PKGS \ - --env AWS_S3_PREFIX_KERNEL_PKGS \ - --env AWS_S3_URI_VIRTUALIZATION \ - --env AWS_S3_URI_MASKING \ - --env AWS_S3_URI_USERLAND_PKGS \ - --env AWS_S3_URI_KERNEL_PKGS \ - --env AWS_S3_URI_LIVEBUILD_ARTIFACTS \ - --env APPLIANCE_PASSWORD \ - --env AWS_ACCESS_KEY_ID \ - --env AWS_SECRET_ACCESS_KEY \ - --env DELPHIX_APPLIANCE_VERSION \ - --env DELPHIX_PACKAGE_MIRROR_MAIN \ - --env DELPHIX_PACKAGE_MIRROR_SECONDARY \ - --env DELPHIX_PLATFORMS \ - --env DELPHIX_SIGNATURE_URL \ - --env DELPHIX_SIGNATURE_TOKEN \ - --env DELPHIX_SIGNATURE_VERSIONS \ - --env DELPHIX_UPGRADE_MINIMUM_VERSION \ - --env DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION \ - --env UPSTREAM_BRANCH="$UPSTREAM_BRANCH" \ - --volume "$TOP:/opt/appliance-build" \ - --workdir "/opt/appliance-build" \ - appliance-build "$@" From 08699cd84a9ee73d26d9fdb0cbf12ca9cf49759d Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 29 Jan 2020 16:02:48 -0500 Subject: [PATCH 020/197] [Backport to 6.0.1.0] Disable apt pipelining for the whole duration of live-build --- .../10-disable-pipeline.chroot_early | 30 +++++++++++++++++++ .../vm-artifacts/84-reset-apt-config.binary | 22 ++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 live-build/config/hooks/configuration/10-disable-pipeline.chroot_early create mode 100755 live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary diff --git a/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early b/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early new file mode 100644 index 00000000..11e2cf09 --- /dev/null +++ b/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early @@ -0,0 +1,30 @@ +#!/bin/bash -eux +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This setting is necessary to prevent the "Undetermined Error" issue when +# downloading packages using apt during the build. chroot_early hooks +# are run after debootstrap and before packages are installed in lb_chroot. +# This setting is reverted in the 84-reset-apt-config.binary hook, after +# all apt operations have been completed. +# +# See https://github.com/delphix/appliance-build/issues/380 for more info +# on the issue. +# +cat <<-EOF >/etc/apt/apt.conf.d/99-delphix-build-disable-pipeline + Acquire::http::Pipeline-Depth 0; +EOF diff --git a/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary b/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary new file mode 100755 index 00000000..e3926d3e --- /dev/null +++ b/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary @@ -0,0 +1,22 @@ +#!/bin/bash -eux +# +# Copyright 2019 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Clear the apt configuration applied for the duration of live-build in the +# 10-disable-pipeline.chroot_early hook. +# +rm binary/etc/apt/apt.conf.d/99-delphix-build-disable-pipeline From c59faa2c874c74127291733b87f3bb1744ffc3c3 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 29 Jan 2020 16:02:48 -0500 Subject: [PATCH 021/197] [Backport to 6.0.0.1] Disable apt pipelining for the whole duration of live-build --- .../10-disable-pipeline.chroot_early | 30 +++++++++++++++++++ .../vm-artifacts/84-reset-apt-config.binary | 22 ++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 live-build/config/hooks/configuration/10-disable-pipeline.chroot_early create mode 100755 live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary diff --git a/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early b/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early new file mode 100644 index 00000000..11e2cf09 --- /dev/null +++ b/live-build/config/hooks/configuration/10-disable-pipeline.chroot_early @@ -0,0 +1,30 @@ +#!/bin/bash -eux +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This setting is necessary to prevent the "Undetermined Error" issue when +# downloading packages using apt during the build. chroot_early hooks +# are run after debootstrap and before packages are installed in lb_chroot. +# This setting is reverted in the 84-reset-apt-config.binary hook, after +# all apt operations have been completed. +# +# See https://github.com/delphix/appliance-build/issues/380 for more info +# on the issue. +# +cat <<-EOF >/etc/apt/apt.conf.d/99-delphix-build-disable-pipeline + Acquire::http::Pipeline-Depth 0; +EOF diff --git a/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary b/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary new file mode 100755 index 00000000..e3926d3e --- /dev/null +++ b/live-build/config/hooks/vm-artifacts/84-reset-apt-config.binary @@ -0,0 +1,22 @@ +#!/bin/bash -eux +# +# Copyright 2019 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Clear the apt configuration applied for the duration of live-build in the +# 10-disable-pipeline.chroot_early hook. +# +rm binary/etc/apt/apt.conf.d/99-delphix-build-disable-pipeline From 902b6fc7b75246f0504ee69cf0c5522f60c43796 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 4 Feb 2020 13:14:32 -0800 Subject: [PATCH 022/197] Remove (now) unused Docker related logic (#405) (#418) --- .../appliance-build.bootstrap/tasks/main.yml | 16 ---- docker/Dockerfile | 87 ----------------- scripts/docker-run.sh | 93 ------------------- 3 files changed, 196 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100755 scripts/docker-run.sh diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 40a77d34..9b067a02 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -15,9 +15,6 @@ # --- -- shell: git rev-parse --show-toplevel 2>/dev/null - register: toplevel - - apt: update_cache: yes @@ -50,16 +47,3 @@ - modprobe: name: zfs state: present - -- apt: - name: - - docker.io - - python3-docker - state: present - -# -# We can't use the docker_image module because it doesn't yet support passing -# the 'network' parameter: https://github.com/ansible/ansible/pull/50313, which -# we need to be able to fetch things from Artifactory. -# -- shell: docker build --network host --tag "appliance-build:latest" "{{ toplevel.stdout }}/docker" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index f2c80968..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,87 +0,0 @@ -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:bionic-20180426 - -MAINTAINER Prakash Surya - -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root - -WORKDIR /root -SHELL ["/bin/bash", "-c"] - -RUN \ - apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:ansible/ansible && \ - apt-get update && \ - apt-get install -y \ - ansible \ - aptly \ - awscli \ - bc \ - coreutils \ - equivs \ - gdisk \ - git \ - java-package \ - jq \ - kpartx \ - libxt6 \ - livecd-rootfs \ - make \ - man \ - openjdk-8-jre-headless \ - pigz \ - rename \ - shellcheck \ - vim \ - zfsutils-linux && \ - rm -rf /var/lib/apt/lists/* - -# -# Download and install Gradle. If downloading from Artifactory doesn't work, -# fall back to downloading from the official Gradle site. This allows us to -# build the Docker image when not on the Delphix network, which is useful for -# running style check via TravisCI. -# -RUN \ - ( wget -nv http://artifactory.delphix.com/artifactory/gradle-distributions/gradle-5.1-bin.zip || \ - wget -nv https://services.gradle.org/distributions/gradle-5.1-bin.zip ) && \ - sha256sum -c <<< '7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d gradle-5.1-bin.zip' && \ - mkdir /opt/gradle && \ - unzip -d /opt/gradle gradle-5.1-bin.zip && \ - rm gradle-5.1-bin.zip - -RUN wget -nv -O /usr/local/bin/shfmt \ - https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_amd64 && \ - chmod +x /usr/local/bin/shfmt - -RUN \ - git clone https://github.com/willthames/ansible-lint /opt/ansible-lint && \ - cd /opt/ansible-lint && \ - git checkout v3.4.21 && \ - git branch -D master -ENV PYTHONPATH="${PYTHONPATH}:/opt/ansible-lint/lib" -ENV PATH="${PATH}:/opt/ansible-lint/bin:/opt/gradle/gradle-5.1/bin" - -# -# Set up the Gradle home directory to be located in a gitignored directory -# inside the repo. This way the cache of downloaded dependencies is preserved -# when the container running a build is destroyed. -# -ENV GRADLE_USER_HOME=/opt/appliance-build/.gradleUserHome diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh deleted file mode 100755 index 733200c4..00000000 --- a/scripts/docker-run.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOP=$(git rev-parse --show-toplevel 2>/dev/null) - -if [[ -z "$TOP" ]]; then - echo "Must be run inside the git repsitory." - exit 1 -fi - -set -o errexit -set -o xtrace - -# -# When running this script interactively (e.g. from an interactive -# terminal session) we want to use the "-ti" options. But, if we use -# these options from a Jenkins job, the command will fail because there -# won't be a TTY available. Thus, we need to check to see if a TTY is -# available before we try to use the "-ti" options. -# -if tty -s; then - DOCKER_RUN="docker run -ti" -else - DOCKER_RUN="docker run" -fi - -# -# Set UPSTREAM_BRANCH. This will determine which version of the linux package -# mirror is used. -# -if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then - echo "UPSTREAM_PRODUCT_BRANCH is not set." - if ! source "$TOP/branch.config" 2>/dev/null; then - echo "No branch.config file found in repo root." - exit 1 - fi - - if [[ -z "$UPSTREAM_BRANCH" ]]; then - echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ - "Ensure branch.config is properly formatted with e.g." \ - "UPSTREAM_BRANCH=\"\"" - exit 1 - fi - echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." -else - UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" -fi -echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" - -$DOCKER_RUN --rm \ - --privileged \ - --network host \ - --ipc "none" \ - --volume /dev:/dev \ - --env AWS_S3_PREFIX_VIRTUALIZATION \ - --env AWS_S3_PREFIX_MASKING \ - --env AWS_S3_PREFIX_USERLAND_PKGS \ - --env AWS_S3_PREFIX_KERNEL_PKGS \ - --env AWS_S3_URI_VIRTUALIZATION \ - --env AWS_S3_URI_MASKING \ - --env AWS_S3_URI_USERLAND_PKGS \ - --env AWS_S3_URI_KERNEL_PKGS \ - --env AWS_S3_URI_LIVEBUILD_ARTIFACTS \ - --env APPLIANCE_PASSWORD \ - --env AWS_ACCESS_KEY_ID \ - --env AWS_SECRET_ACCESS_KEY \ - --env DELPHIX_APPLIANCE_VERSION \ - --env DELPHIX_PACKAGE_MIRROR_MAIN \ - --env DELPHIX_PACKAGE_MIRROR_SECONDARY \ - --env DELPHIX_PLATFORMS \ - --env DELPHIX_SIGNATURE_URL \ - --env DELPHIX_SIGNATURE_TOKEN \ - --env DELPHIX_SIGNATURE_VERSIONS \ - --env DELPHIX_UPGRADE_MINIMUM_VERSION \ - --env DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION \ - --env UPSTREAM_BRANCH="$UPSTREAM_BRANCH" \ - --volume "$TOP:/opt/appliance-build" \ - --workdir "/opt/appliance-build" \ - appliance-build "$@" From 7b9aa6a9337f348865d29a5c9ce9fc30faa96d74 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 13 Feb 2020 07:19:09 -0800 Subject: [PATCH 023/197] DLPX-67625 [Backport of Issue DLPX-67598 to 6.0.1.0] VDBs becomes inactive during some deferred upgrades from 6.0.1.0, similar to DLPX-66733 (#427) * DLPX-66981 Add verbose logging to the upgrade bash scripts (#397) * Log upgrade scripts' stdout and stderr via syslog (#414) This change builds on the prior commit 9ba7066. Now, in addition to logging the upgrade scripts' execution trace to syslog, we allow scripts to log the stdout and stderr of their commands to syslog as well. This new functionality is leveraged in the "execute" script, such that we keep a better log of what "apt-get" is doing when it's upgrading packages on the system. * Avoid removing "etab" file on upgrade (#421) --- upgrade/upgrade-scripts/common.sh | 83 ++++++++++++++++++++++++++++++- upgrade/upgrade-scripts/execute | 42 ++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index d1b5c1a2..bbd20842 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -39,13 +39,92 @@ LOG_DIRECTORY="/var/tmp/delphix-upgrade" PROP_CURRENT_VERSION="com.delphix:current-version" PROP_INITIAL_VERSION="com.delphix:initial-version" +# +# To better enable root cause analysis of any upgrade failures, we +# enable the "xtrace" feature here, and redirect that output to the +# system log. This way, we can easily obtain a trace of the execution +# path via the system log, which can be invaluable for any post-mortem +# analysis of a failure. Verbose logging is available in /var/log/syslog +# on the Delphix Engine +# +exec 4> >(logger -t "upgrade-scripts:$(basename "$0")" --id=$$) +BASH_XTRACEFD="4" +PS4='${BASH_SOURCE[0]}:${FUNCNAME[0]}:${LINENO[0]} ' +set -o xtrace + +# +# In addition to redirecting the execution trace output to syslog (which +# is configured above), we also provide the following functions such +# that each script can enable and disable the redirection of their +# "stdout" and "stderr" to that same system log. This way, for the +# scripts that leverage these functions, we'll capture a trace of the +# script's execution, along with the output of the executed commands, in +# a single location (complete with timestamps for all executed commands +# and the commands' output). +# +# We don't automatically enable this redirection, since it would then +# mask usage and help messages that can be helpful when manually running +# the scripts. Thus, the intention is for each script to determine when +# it's most appropriate to enable and disable this redirection. +# + +function start_stdout_redirect_to_system_log() { + exec 5>&1 + exec 1>&4 +} + +function stop_stdout_redirect_to_system_log() { + exec 1>&5 +} + +# +# This global variable is used to track which file descriptor +# corresponds to the script's stderr. This is relevant if a script +# redirects its stderr to the system log using the functions below, and +# helps us ensure errors (i.e. any calls to "die", "warn", etc.) will +# always be visible on stderr. +# +STDERR_FD=2 + +function start_stderr_redirect_to_system_log() { + STDERR_FD=6 + eval "exec $STDERR_FD>&2" + exec 2>&4 +} + +function stop_stderr_redirect_to_system_log() { + exec 2>&$STDERR_FD + STDERR_FD=2 +} + function die() { - echo "$(basename "$0"): $*" >&2 + echo "$(basename "$0"): $*" >&$STDERR_FD + + if [[ "$STDERR_FD" != "2" ]]; then + # + # If stderr is configured to be redirected to syslog, we + # want to emit the error message to both, syslog and the + # script's actual stderr file descriptor; this ensures + # the message is sent to syslog too. + # + echo "$(basename "$0"): $*" >&2 + fi + exit 1 } function warn() { - echo "$(basename "$0"): $*" >&2 + echo "$(basename "$0"): $*" >&$STDERR_FD + + if [[ "$STDERR_FD" != "2" ]]; then + # + # If stderr is configured to be redirected to syslog, we + # want to emit the error message to both, syslog and the + # script's actual stderr file descriptor; this ensures + # the message is sent to syslog too. + # + echo "$(basename "$0"): $*" >&2 + fi } function get_image_path() { diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index a3040959..1f37ab5b 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -133,6 +133,45 @@ deb [trusted=yes] file://$IMAGE_PATH/public bionic delphix EOF die "failed to configure apt sources" +start_stdout_redirect_to_system_log +start_stderr_redirect_to_system_log + +# +# Older versions (i.e. the 6.0.0.0 release) of the "nfs-kernel-server" +# package had "etab" file delivered as part of the package. Thus, when +# upgrading the package, the existing "etab" file would get replaced +# with a new/empty "etab" file from the new package. +# +# This file is dynamically modified during runtime, to contain the list +# of currently exported filesystems. Thus, when it's replaced on upgrade +# with an empty file, this results in all exports being unexported; i.e. +# mountd notices that the file is empty, interprets that to mean no +# filesystems should be currently exported, and then unexports all +# previously exported filesystems. +# +# This is problematic, since the act of unexporting the filesystems can +# result in client errors, if those exports are being actively used at +# the time of the upgrade/unexport. +# +# To avoid this issue, we're modifying the dpkg state, such that dpkg +# will no longer associate the "etab" file with the "nfs-kernel-server" +# package. This way, when the old package is removed, the "etab" file +# will not be automatically manipulated by dpkg. We *must* do this +# prior to invoking "apt-get" below, to ensure we remove the association +# before upgrading the package. +# +# FWIW, we tried to do this via the package's "preinst" package hook, +# but that did not help; the "etab" file was still removed when the old +# package was replaced/upgraded. +# +# We can safely remove this once running a version that no longer +# supports upgrading from the 6.0.0.0 release, since that's the only +# release that suffers from this defect; we've dis-associated the "etab" +# file from the "nfs-kernel-server" package in all later releases. +# +[[ -e /var/lib/dpkg/info/nfs-kernel-server.list ]] && + sed -i '/\/var\/lib\/nfs\/etab/d' /var/lib/dpkg/info/nfs-kernel-server.list + apt_get update || die "failed to update apt sources" # @@ -284,6 +323,9 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # apt_get autoremove --purge -y || die "autoremove after upgrade failed" +stop_stdout_redirect_to_system_log +stop_stderr_redirect_to_system_log + # # Finally, for all of the packages listed in the "packages.list" file, # we verify the package is installed and its version is correct; this From abaa7acd4fa3153e0f8ce00fd824f6b357b8d494 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 13 Feb 2020 07:22:51 -0800 Subject: [PATCH 024/197] DLPX-66509 [Backport of Issue DLPX-66313 to 6.0.1.0] files previously installed into "/etc" by packages may not be removed on upgrade (#428) * Remove obsolete configuration files on upgrade (#425) --- upgrade/upgrade-scripts/execute | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 1f37ab5b..6777fa90 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -323,6 +323,56 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # apt_get autoremove --purge -y || die "autoremove after upgrade failed" +# +# Package configuration files are only automatically removed by the +# package manager when the package that "owns" the file is "purged". +# Thus, when upgrading a package to a new version that no longer +# delivers a "conffile" that it used to deliver, that conffile will not +# be automatically removed by the package manager. +# +# Generally, it is the responsibility of each specific package to +# properly remove the conffile that is no longer used by the package. +# Unfortuantely this means that if a given package does not do this +# removal, the conffile will remain on the system indefinitely. Further, +# that configuration file may actually result in a mis-configured +# system. +# +# To help avoid mis-configuration when packages mistakenly omit the +# removal of their conffiles, we have logic below to automatically +# detect these "obsolete" conffiles, and remove them proactively. +# Further, after we remove these obsolete files, we "--reinstall" the +# package that previously owned the file, such that the package manager +# will no longer report that file as obsolete. +# +dpkg-query -Wf '${Conffiles}\n' | awk '$3 == "obsolete" {print $1}' | + while read -r file; do + package=$(dpkg-query -S "$file" | awk -F: '{print $1}') + [[ -n "$package" ]] || die "failed to determine package name" + + # + # We should only be operating on currently installed + # packages. So, to verify the package retrieved above, + # we double check that the package is installed. + # + dpkg-query -W "$package" &>/dev/null || + die "package '$package' is not installed" + + rm -f "$file" || + die "failed to remove file '$file' of package '$package'" + + # + # Since we run this each time a file is removed, if a + # package has multiple obsolete files, we'll end up + # reinstalling the package multiple times. Currently + # this isn't a problem, since we haven't hit a case + # where a package has multiple obsolete files; but + # perhaps this will change in the future, at which point + # we can optimize this better to handle that case. + # + apt_get install -y --reinstall "$package" || + die "failed to reinstall package '$package'" + done || die "failed to remove obsolete package configuration files" + stop_stdout_redirect_to_system_log stop_stderr_redirect_to_system_log From 5dbcdf6eccf4a9899903ab7b5248be778478ea8f Mon Sep 17 00:00:00 2001 From: George Wilson Date: Sat, 15 Feb 2020 17:05:58 -0700 Subject: [PATCH 025/197] [Backport of DLPX-68440 to 6.0.1.0] remove server.conf and demo image from appliance-build (#429) --- .../tasks/main.yml | 23 ---------------- .../tasks/main.yml | 23 ---------------- .../tasks/main.yml | 23 ---------------- .../tasks/main.yml | 23 ---------------- .../tasks/main.yml | 23 ---------------- .../external-demo/ansible/playbook.yml | 27 ------------------- .../variants/external-demo/ansible/roles | 1 - .../external-standard/ansible/playbook.yml | 2 -- .../internal-dev/ansible/playbook.yml | 2 -- .../variants/internal-qa/ansible/playbook.yml | 2 -- 10 files changed, 149 deletions(-) delete mode 100644 live-build/misc/ansible-roles/appliance-build.virtualization-demo/tasks/main.yml delete mode 100644 live-build/misc/ansible-roles/appliance-build.virtualization-external/tasks/main.yml delete mode 100644 live-build/misc/ansible-roles/appliance-build.virtualization-internal/tasks/main.yml delete mode 100644 live-build/misc/ansible-roles/appliance-build.virtualization-standard/tasks/main.yml delete mode 100644 live-build/variants/external-demo/ansible/playbook.yml delete mode 120000 live-build/variants/external-demo/ansible/roles diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml index af1fb83c..224ff921 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml @@ -23,29 +23,6 @@ path: /var/opt/delphix state: directory -# -# The "product", "platform", and "cr_auth" fields are intentionally left -# out of this file, as those fields should get filled in by other roles -# that will run after this role. -# -- copy: - dest: /var/opt/delphix/server.conf - mode: 0644 - content: | - [config] - hostname=delphix - disk=c?t0d0 - interface=vmxnet3s0 - primary_aggr_nic=vmxnet3s0 - type=ipv4 - method=dhcp - ip= - netmask= - gateway= - domain= - dns1= - dns2= - # # Initial NFS Server configuration for virtualization. We do this here instead # of within the virtualization package because the virtualization software diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-demo/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-demo/tasks/main.yml deleted file mode 100644 index 00798570..00000000 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-demo/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- lineinfile: - path: /var/opt/delphix/server.conf - regexp: "^{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - { key: 'product', value: 'demo' } diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-external/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-external/tasks/main.yml deleted file mode 100644 index c3f44d0a..00000000 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-external/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- lineinfile: - path: /var/opt/delphix/server.conf - regexp: "^{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - { key: 'cr_auth', value: 'true' } diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-internal/tasks/main.yml deleted file mode 100644 index caba87db..00000000 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-internal/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- lineinfile: - path: /var/opt/delphix/server.conf - regexp: "^{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - { key: 'cr_auth', value: 'false' } diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-standard/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-standard/tasks/main.yml deleted file mode 100644 index 2c89ce2d..00000000 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-standard/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- lineinfile: - path: /var/opt/delphix/server.conf - regexp: "^{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - { key: 'product', value: 'standard' } diff --git a/live-build/variants/external-demo/ansible/playbook.yml b/live-build/variants/external-demo/ansible/playbook.yml deleted file mode 100644 index 11358e04..00000000 --- a/live-build/variants/external-demo/ansible/playbook.yml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- hosts: all - gather_facts: no - vars: - ansible_python_interpreter: /usr/bin/python3 - roles: - - appliance-build.minimal-common - - appliance-build.masking-common - - appliance-build.virtualization-common - - appliance-build.virtualization-external - - appliance-build.virtualization-demo diff --git a/live-build/variants/external-demo/ansible/roles b/live-build/variants/external-demo/ansible/roles deleted file mode 120000 index 58b96301..00000000 --- a/live-build/variants/external-demo/ansible/roles +++ /dev/null @@ -1 +0,0 @@ -../../../misc/ansible-roles \ No newline at end of file diff --git a/live-build/variants/external-standard/ansible/playbook.yml b/live-build/variants/external-standard/ansible/playbook.yml index c44df9a0..32d2dae7 100644 --- a/live-build/variants/external-standard/ansible/playbook.yml +++ b/live-build/variants/external-standard/ansible/playbook.yml @@ -23,5 +23,3 @@ - appliance-build.minimal-common - appliance-build.masking-common - appliance-build.virtualization-common - - appliance-build.virtualization-external - - appliance-build.virtualization-standard diff --git a/live-build/variants/internal-dev/ansible/playbook.yml b/live-build/variants/internal-dev/ansible/playbook.yml index 2bd1baa4..bac20acb 100644 --- a/live-build/variants/internal-dev/ansible/playbook.yml +++ b/live-build/variants/internal-dev/ansible/playbook.yml @@ -39,7 +39,5 @@ - appliance-build.masking-development - appliance-build.qa-internal - appliance-build.virtualization-common - - appliance-build.virtualization-internal - - appliance-build.virtualization-standard - appliance-build.virtualization-development - appliance-build.zfsonlinux-development diff --git a/live-build/variants/internal-qa/ansible/playbook.yml b/live-build/variants/internal-qa/ansible/playbook.yml index fc53056c..21df7ffc 100644 --- a/live-build/variants/internal-qa/ansible/playbook.yml +++ b/live-build/variants/internal-qa/ansible/playbook.yml @@ -25,5 +25,3 @@ - appliance-build.masking-common - appliance-build.qa-internal - appliance-build.virtualization-common - - appliance-build.virtualization-internal - - appliance-build.virtualization-standard From f75da6432185aa6660c624c598ce657078d7d77e Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 20 Feb 2020 12:57:47 -0800 Subject: [PATCH 026/197] DLPX-68603 [Backport of DLPX-68595 to 6.0.1.0] delphix-fluentd service disabled during upgrade: fix for 6.0.0.0 (#436) --- upgrade/upgrade-scripts/execute | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 6777fa90..235b314d 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -172,6 +172,18 @@ start_stderr_redirect_to_system_log [[ -e /var/lib/dpkg/info/nfs-kernel-server.list ]] && sed -i '/\/var\/lib\/nfs\/etab/d' /var/lib/dpkg/info/nfs-kernel-server.list +# +# Older versions (i.e. the 6.0.0.0 release) of the virtualization +# package would disable the "delphix-fluentd" service in that package's +# "prerm" package hook. This meant that if the service was enabled prior +# to the upgrade, it would be disabled after the upgrade. Since we can't +# easily stop this behavior on systems already running 6.0.0.0, we have +# to workaround this issue. Thus, before we upgrade the packages below, +# we check to see if this service is currently enabled, and will +# re-enable it after all packages have been upgraded. +# +DELPHIX_FLUENTD_IS_ENABLED=$(systemctl is-enabled delphix-fluentd.service) + apt_get update || die "failed to update apt sources" # @@ -323,6 +335,17 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # apt_get autoremove --purge -y || die "autoremove after upgrade failed" +# +# As mentioned in a comment above, if the "delphix-fluentd" service is +# enabled prior to upgrading all of the packages, we need to ensure it +# remains enabled after upgrading all of the packages. Due to a bug in +# 6.0.0.0 this might not be the case, so we explicitly enable it here. +# +if [[ "$DELPHIX_FLUENTD_IS_ENABLED" == "enabled" ]]; then + systemctl enable "delphix-fluentd.service" || + die "failed to enable 'delphix-fluend.service'" +fi + # # Package configuration files are only automatically removed by the # package manager when the package that "owns" the file is "purged". From 7868fcdad6b620549fec13aee137c4429d46e92f Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 21 Feb 2020 13:52:15 -0500 Subject: [PATCH 027/197] [Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0 --- .../hooks/vm-artifacts/90-linux-migration-artifact.binary | 2 +- live-build/misc/migration-scripts/dx_apply | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary index 269a508f..7e6ec5ec 100755 --- a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary +++ b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary @@ -114,7 +114,7 @@ test -n "$DELPHIX_APPLIANCE_VERSION" # of migration, so every time we bump this number we # need to bump the one from the checks there too. # - echo "DLPX_MIN_VERSION=5.3.6.0" + echo "DLPX_MIN_VERSION=5.3.7.0" # # DLPX_VERSION is set explicitly to match the version of the diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 75efc62b..0b355700 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -73,7 +73,7 @@ ARCHIVE_DIR="$1" # [[ -n "$DLPX_VERSION" ]] || die "DLPX_VERSION variable is missing" -MIN_MIGRATION_VERSION="5.3.6.0" +MIN_MIGRATION_VERSION="5.3.7.0" [[ "$DLPX_MIN_VERSION" == "$MIN_MIGRATION_VERSION" ]] || die "expected DLPX_MIN_VERSION for migration to be" \ "$MIN_MIGRATION_VERSION" From dabe56dc4a623885efa609a0862b7078501df66f Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 10 Mar 2020 10:12:50 -0700 Subject: [PATCH 028/197] DLPX-68848 [Backport of DLPX-68832 to 6.0.2.0] DFE during trunk to trunk deferred upgrade, Failed to set-bootfs for container 'delphix.AvZ8Xle'. (#443) --- upgrade/upgrade-scripts/rootfs-container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/rootfs-container b/upgrade/upgrade-scripts/rootfs-container index da0cb4fe..14c9f4d4 100755 --- a/upgrade/upgrade-scripts/rootfs-container +++ b/upgrade/upgrade-scripts/rootfs-container @@ -77,7 +77,7 @@ function get_bootloader_devices() { # zpool list -vH rpool | awk '! /rpool|mirror|replacing|spare/ {print $1}' | - sed 's/[0-9]*$//' + sed 's/p\{0,1\}[0-9]*$//' } function set_bootfs_not_mounted_cleanup() { From 9e50fa5f8d627e582de581e8e5837397e329337d Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 12 Mar 2020 15:00:20 -0400 Subject: [PATCH 029/197] Revert "[Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0" This reverts commit 7868fcdad6b620549fec13aee137c4429d46e92f. --- .../hooks/vm-artifacts/90-linux-migration-artifact.binary | 2 +- live-build/misc/migration-scripts/dx_apply | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary index 7e6ec5ec..269a508f 100755 --- a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary +++ b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary @@ -114,7 +114,7 @@ test -n "$DELPHIX_APPLIANCE_VERSION" # of migration, so every time we bump this number we # need to bump the one from the checks there too. # - echo "DLPX_MIN_VERSION=5.3.7.0" + echo "DLPX_MIN_VERSION=5.3.6.0" # # DLPX_VERSION is set explicitly to match the version of the diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 0b355700..75efc62b 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -73,7 +73,7 @@ ARCHIVE_DIR="$1" # [[ -n "$DLPX_VERSION" ]] || die "DLPX_VERSION variable is missing" -MIN_MIGRATION_VERSION="5.3.7.0" +MIN_MIGRATION_VERSION="5.3.6.0" [[ "$DLPX_MIN_VERSION" == "$MIN_MIGRATION_VERSION" ]] || die "expected DLPX_MIN_VERSION for migration to be" \ "$MIN_MIGRATION_VERSION" From a49d8c6c8b18fb64a17a2ce390469070f89a9479 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 5 Mar 2020 12:38:18 -0500 Subject: [PATCH 030/197] DLPX-68826 [Backport of DLPX-68825 to 6.0.2.0] delphix-fluentd service disabled after not-in-place upgrade --- upgrade/upgrade-scripts/upgrade-container | 1 + 1 file changed, 1 insertion(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index cb6273d2..540e4982 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -594,6 +594,7 @@ function migrate_configuration() { while read -r svc; do migrate_svc "$svc" done <<-EOF + delphix-fluentd.service delphix-masking.service nfs-mountd.service ntp.service From b0fec5f1d8353654ac952a68d186f1ab527801ef Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 24 Mar 2020 07:53:20 -0700 Subject: [PATCH 031/197] DLPX-68769 [Backport of DLPX-68763 to 6.0.2.0] crashdump quota on rpool gets reset on upgrade (#453) --- .../hooks/vm-artifacts/90-raw-disk-image.binary | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 3efa513e..a68e0d57 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -204,16 +204,13 @@ zfs create \ # # Initialize the crashdump dataset. This is used to store core files -# from processes that have crashed. The configuration of the crashdump -# dataset is split between appliance-build and delphix-platform. -# The main reason for the split is that on initial image creation, -# delphix-platform gets installed prior to the creation of rpool so we -# can't create the zfs dataset from that package. We instead do the dataset -# creation here and allow delphix-platform to reconfigure the service on -# first boot or upgrade. +# from processes that have crashed. Since we don't have control on how +# many of these core files accumulate, we set a reasonable quota (50% of +# the rpool's size) to keep these from running the rpool out of space. # zfs create \ -o mountpoint=legacy \ + -o quota="$(echo "$(zpool list -Hpo size rpool) / 2" | bc)b" \ rpool/crashdump # From 870c2a85a024a45f9cfae9e5dbd4644f88cbcace Mon Sep 17 00:00:00 2001 From: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Date: Mon, 30 Mar 2020 15:09:13 -0700 Subject: [PATCH 032/197] Backports of 4 commits to add 'upgrade finalize' to 6.0/stage (#457) * [Backport to 6.0.2.0] Add new "finalize" subcommand to upgrade script (#413) This change adds the "finalize" subcommand to the upgrade script, which is intended to be used to clean up any filesystems, snapshots, etc. that may have been generated during the upgrade process (e.g. for rollback). This is intended to be used to mark the finalization or completion of the upgrade, after which rollback will no longer be possible; e.g. $ sudo /var/dlpx-update/latest/upgrade deferred $ sudo /var/dlpx-update/latest/upgrade finalize * [Backport to 6.0.2.0] "rootfs-container delete" must process "log" dataset (#448) When "/var/log" was moved to a seperate dataset in commit 76a37744, we failed to properly adapt the "rootfs-container delete" script to account for this change. Since then, the "rootfs-container delete" script has likely been broken; currently, when I attempt to use this script, it fails with the following error: $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.xfXCbRc cannot destroy 'rpool/ROOT/delphix.xfXCbRc': filesystem has dependent clones use '-R' to destroy the following datasets: rpool/ROOT/delphix.iluoSLh/log rootfs-container: 'zfs destroy -r rpool/ROOT/delphix.xfXCbRc' failed The problem is, we're not properly handling the "log" dataset clones. The fix is simple, we need to add the "log" dataset to the list of datasets that we account for (i.e. the datasets that may need to be "zfs promote"-ed) when calling "rootfs-container delete". * [Backport to 6.0.2.0] Remove lingering snapshots after rollback event (#449) After performing a rollback of a failed not-in-place upgrade, if we remove the container associated with that failure (i.e. the "new" container), we will not remove the "container-delphix.XXXXXXX" snapshots used to generate that new container, and these snapshots will persist indefinitely. This change adds the necessary logic to the "rootfs-container delete" script to remove these snapshots, after removing the container itself. For example, without this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.Bs4VTPo $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU@container-delphix.Bs4VTPo 0B - 64K - rpool/ROOT/delphix.jDHoOtU/data 34.1M 44.3G 33.1M legacy rpool/ROOT/delphix.jDHoOtU/data@container-delphix.Bs4VTPo 1006K - 30.5M - rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/home@container-delphix.Bs4VTPo 151K - 11.8G - rpool/ROOT/delphix.jDHoOtU/log 28.7M 44.3G 27.0M legacy rpool/ROOT/delphix.jDHoOtU/log@container-delphix.Bs4VTPo 1.61M - 3.95M - rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / rpool/ROOT/delphix.jDHoOtU/root@container-delphix.Bs4VTPo 2.02M - 5.86G - And now, with this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.lJvMWgM $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU/data 34.5M 44.3G 34.5M legacy rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/log 27.1M 44.3G 27.1M legacy rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / Thus, with this change, the "container-delphix.XXXXXXX" snapshots are properly removed after calling "rootfs-container delete". * DLPX-69176 [Backport of DLPX-68157 to 6.0.2.0] Update 'finalize' logic to clean up snapshots older than current and previous Co-authored-by: Prakash Surya --- upgrade/upgrade-scripts/common.sh | 11 +++ upgrade/upgrade-scripts/rootfs-container | 22 ++++-- upgrade/upgrade-scripts/upgrade | 99 +++++++++++++++++++++++- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index bbd20842..a6f01996 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -263,6 +263,11 @@ function source_upgrade_properties() { die "failed to source: '$UPDATE_DIR/upgrade.properties'" } +function remove_upgrade_properties() { + rm "$UPDATE_DIR/upgrade.properties" || + die "failed to remove: '$UPDATE_DIR/upgrade.properties'" +} + function set_upgrade_property() { [[ -n "$1" ]] || die "upgrade property key is missing" [[ -n "$2" ]] || die "upgrade property value is missing" @@ -283,3 +288,9 @@ function set_upgrade_property() { source_upgrade_properties || die "failed to read properties file after setting '$1=$2'" } + +function verify_upgrade_not_in_progress() { + . "$UPDATE_DIR/upgrade.properties" + + [[ -z "$UPGRADE_TYPE" ]] || die "upgrade currently in-progress" +} diff --git a/upgrade/upgrade-scripts/rootfs-container b/upgrade/upgrade-scripts/rootfs-container index 14c9f4d4..f2af998f 100755 --- a/upgrade/upgrade-scripts/rootfs-container +++ b/upgrade/upgrade-scripts/rootfs-container @@ -31,16 +31,17 @@ function delete() { local clonesnaps=() # - # The "data" and "home" datasets of a rootfs container may have - # been cloned as part of a prior upgrade, and the "root" dataset - # may have been cloned as part of a prior rollback. Thus, in - # order to delete this specific rootfs container, we need to - # promote any clones that exist. + # The "data", "home", and "log" datasets of a rootfs container + # may have been cloned as part of a prior upgrade, and the + # "root" dataset may have been cloned as part of a prior + # rollback. Thus, in order to delete this specific rootfs + # container, we need to promote any clones that exist. # for snap in \ $(get_dataset_snapshots "rpool/ROOT/$CONTAINER/root") \ $(get_dataset_snapshots "rpool/ROOT/$CONTAINER/data") \ - $(get_dataset_snapshots "rpool/ROOT/$CONTAINER/home"); do + $(get_dataset_snapshots "rpool/ROOT/$CONTAINER/home") \ + $(get_dataset_snapshots "rpool/ROOT/$CONTAINER/log"); do for clone in $(get_snapshot_clones "$snap"); do zfs promote "$clone" || die "'zfs promote $clone' failed" @@ -62,6 +63,15 @@ function delete() { for snap in "${clonesnaps[@]}"; do zfs destroy "$snap" || die "'zfs destroy $snap' failed" done + + ROOTFS_DATASET=$(get_mounted_rootfs_container_dataset) + [[ -n "$ROOTFS_DATASET" ]] || + die "unable to determine mounted rootfs container dataset" + + if zfs list "$ROOTFS_DATASET@container-$CONTAINER" &>/dev/null; then + zfs destroy -r "$ROOTFS_DATASET@container-$CONTAINER" || + die "failed to destroy container snapshot: '$CONTAINER'" + fi } function get_bootloader_devices() { diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 791b80a0..7c88cb17 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -54,6 +54,7 @@ function usage() { echo "$PREFIX_STRING [-n] [-v] deferred" echo "$PREFIX_SPACES [-n] [-v] full" echo "$PREFIX_SPACES rollback" + echo "$PREFIX_SPACES finalize" exit 2 } @@ -296,7 +297,9 @@ function rollback() { source_upgrade_properties - [[ -n "$UPGRADE_TYPE" ]] || die "variable UPGRADE_TYPE is not set" + [[ -n "$UPGRADE_TYPE" ]] || + die "variable UPGRADE_TYPE is not set; is upgrade in progress?" + [[ -n "$UPGRADE_BASE_CONTAINER" ]] || die "variable UPGRADE_BASE_CONTAINER is not set" [[ -n "$UPGRADE_BASE_VERSION" ]] || @@ -357,6 +360,94 @@ function rollback() { systemctl reboot || die "'systemctl reboot' failed" } +function finalize() { + if [[ "$DLPX_UPGRADE_DRY_RUN" == "true" ]]; then + # + # If we're executing a dry-run of the finalize + # sub-command, we assume that we're running after a + # dry-run upgrade. Thus, in that case, we only need to + # remove the upgrade image, as all of the snapshots and + # datasets for the dry-run upgrade will have already + # been removed. + # + rm -rf "$IMAGE_PATH" || + die "failed to remove unpacked upgrade image" + return + fi + + source_upgrade_properties + + [[ -n "$UPGRADE_TYPE" ]] || + die "variable UPGRADE_TYPE is not set; is upgrade in progress?" + [[ -n "$UPGRADE_BASE_VERSION" ]] || + die "variable UPGRADE_BASE_VERSION is not set" + + case "$UPGRADE_TYPE" in + DEFERRED | FULL | ROLLBACK) ;; + *) + die "finalize is not supported for upgrade type: '$UPGRADE_TYPE'" + ;; + esac + + # + # This first pass checks for available "execute.upgrade" snapshots + # and deletes snapshots that are older than the base upgrade version. + # This pass takes care of cleanup after in-place and deferred upgrades. + # + SNAPSHOT_LIST=$(zfs list -t snapshot -r rpool/ROOT -d 2 -Ho name) || + die "unable to determine available rollback snapshots" + + for snapshot in $SNAPSHOT_LIST; do + local SNAPSHOT_NAME + SNAPSHOT_NAME=$(echo "$snapshot" | awk -F@ '{print $2}') + # + # We skip container-delphix instances since `rootfs-container delete` + # handles them in the second pass below. + # + grep -qE "^container-delphix.[[:alnum:]]{7}$" <(echo "$SNAPSHOT_NAME") && + continue + grep -qE "^execute-upgrade.[[:alnum:]]{7}$" <(echo "$SNAPSHOT_NAME") || + die "unexpected snapshot name: '$snapshot'" + local SNAPSHOT_VERSION + SNAPSHOT_VERSION=$(zfs get -Hpo value "$PROP_CURRENT_VERSION" "$snapshot") || + die "failed to get snapshot version" + [[ "$SNAPSHOT_VERSION" != "-" ]] || + die "failed to get current version for snapshot '$snapshot'" + if compare_versions \ + "$SNAPSHOT_VERSION" "lt" "$UPGRADE_BASE_VERSION"; then + zfs destroy -r "$snapshot" || + die "failed to destroy rollback snapshot" + fi + done + + # + # This second pass checks for available filesystems and deletes those + # that are older than the base upgrade version. This pass takes care + # of cleanup following rollbacks and not-in-place upgrades. + # + FILESYSTEM_LIST=$(zfs list -t filesystem -r rpool/ROOT -d 1 -Ho name | tail -n +2) || + die "unable to determine available filesystems" + + for filesystem in $FILESYSTEM_LIST; do + local FILESYSTEM_VERSION + FILESYSTEM_VERSION=$(zfs get -Hpo value "$PROP_CURRENT_VERSION" "$filesystem") || + die "failed to get filesystem version" + [[ "$FILESYSTEM_VERSION" != "-" ]] || + die "failed to get current version for filesystem '$filesystem'" + if compare_versions \ + "$FILESYSTEM_VERSION" "lt" "$UPGRADE_BASE_VERSION"; then + local FILESYSTEM_NAME + FILESYSTEM_NAME=$(echo "$filesystem" | awk -F/ '{print $3}') + "$IMAGE_PATH/rootfs-container" delete "$FILESYSTEM_NAME" || + die "failed to delete filesystem '$filesystem" + fi + done + + rm -rf "$IMAGE_PATH" || die "failed to remove unpacked upgrade image" + + remove_upgrade_properties +} + [[ "$EUID" -ne 0 ]] && die "must be run as root" while getopts ':nv' c; do @@ -380,12 +471,14 @@ deferred) shift 1 verify_upgrade_is_allowed verify_upgrade_in_place_is_allowed + verify_upgrade_not_in_progress upgrade_in_place "$@" ;; full) UPGRADE_TYPE="FULL" shift 1 verify_upgrade_is_allowed + verify_upgrade_not_in_progress # # FULL upgrade always perform a reboot but can take on two @@ -405,6 +498,10 @@ rollback) shift 1 rollback "$@" ;; +finalize) + shift 1 + finalize "$@" + ;; *) usage "invalid option -- '$1'" ;; From 690deb80c3528810997cc092b03605dba262d9b5 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Mon, 6 Apr 2020 09:12:26 -0700 Subject: [PATCH 033/197] Use "delphix-upgrade-verification" package in "verify-jar" script (#455) (#463) DLPX-64198 define and implement version info file for verification package DLPX-68919 Use New upgrade-verify.jar in appliance-build DLPX-69031 Make sure that the delphix-verification package is not installed post upgrade --- build.gradle | 5 +- scripts/build-ancillary-repository.sh | 54 +---------------- scripts/build-upgrade-image.sh | 33 ++++++++++- scripts/common.sh | 68 ++++++++++++++++++++++ upgrade/upgrade-scripts/common.sh | 14 +++++ upgrade/upgrade-scripts/execute | 14 ----- upgrade/upgrade-scripts/verify-jar | 25 +++++++- upgrade/verification-version.info.template | 26 +++++++++ upgrade/version.info.template | 5 ++ 9 files changed, 173 insertions(+), 71 deletions(-) create mode 100644 scripts/common.sh create mode 100644 upgrade/verification-version.info.template diff --git a/build.gradle b/build.gradle index c670dfa3..74fcef90 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,10 @@ for (variant in allVariants) { } } - for (envVar in ["DELPHIX_PLATFORMS", "AWS_S3_URI_LIVEBUILD_ARTIFACTS"]) { + for (envVar in ["DELPHIX_PLATFORMS", + "AWS_S3_URI_LIVEBUILD_ARTIFACTS", + "AWS_S3_URI_UPGRADE_VERIFICATION", + "AWS_S3_PREFIX_UPGRADE_VERIFICATION"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) } diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index f3000f35..d487a06a 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -15,6 +15,8 @@ # limitations under the License. # +. "${BASH_SOURCE%/*}/common.sh" + # # This script is intended to be used to build the "ancillary" repository # that is used when we run live-build to build our artifacts. Prior to @@ -29,8 +31,6 @@ # commands (e.g. apt install, apt download, etc). # -TOP=$(git rev-parse --show-toplevel 2>/dev/null) - if [[ -z "$TOP" ]]; then echo "Must be run inside the git repsitory." 2>&1 exit 1 @@ -42,56 +42,6 @@ set -o pipefail OUTPUT_DIR=$TOP/live-build/build/ancillary-repository -function resolve_s3_uri() { - local pkg_uri="$1" - local pkg_prefix="$2" - local latest_subprefix="$3" - - local bucket="snapshot-de-images" - local jenkinsid="jenkins-ops" - local resolved_uri - - if [[ -n "$pkg_uri" ]]; then - resolved_uri="$pkg_uri" - elif [[ "$pkg_prefix" == s3* ]]; then - resolved_uri="$pkg_prefix" - elif [[ -n "$pkg_prefix" ]]; then - resolved_uri="s3://$bucket/$pkg_prefix" - elif [[ -n "$latest_subprefix" ]]; then - aws s3 cp --quiet \ - "s3://$bucket/builds/$jenkinsid/$latest_subprefix" . - resolved_uri="s3://$bucket/$(cat latest)" - rm -f latest - else - echo "Invalid arguments provided to resolve_s3_uri()" 2>&1 - exit 1 - fi - - if aws s3 ls "$resolved_uri" &>/dev/null; then - echo "$resolved_uri" - else - echo "'$resolved_uri' not found." 1>&2 - exit 1 - fi -} - -function download_delphix_s3_debs() { - local pkg_directory="$1" - local S3_URI="$2" - local tmp_directory - - tmp_directory=$(mktemp -d -p "$TOP/build" tmp.s3-debs.XXXXXXXXXX) - pushd "$tmp_directory" &>/dev/null - - aws s3 sync --only-show-errors "$S3_URI" . - sha256sum -c --strict SHA256SUMS - - mv ./*deb "$pkg_directory/" - - popd &>/dev/null - rm -rf "$tmp_directory" -} - function build_ancillary_repository() { local pkg_directory="$1" diff --git a/scripts/build-upgrade-image.sh b/scripts/build-upgrade-image.sh index f974f190..fdfbd002 100755 --- a/scripts/build-upgrade-image.sh +++ b/scripts/build-upgrade-image.sh @@ -15,6 +15,8 @@ # limitations under the License. # +. "${BASH_SOURCE%/*}/common.sh" + # # This script is intended to build an upgrade image that contains all of # the packages needed to upgrade a particular variant of the appliance, @@ -26,8 +28,6 @@ # variant (of which there will be one per supported platform). # -TOP=$(git rev-parse --show-toplevel 2>/dev/null) - if [[ -z "$TOP" ]]; then echo "Must be run inside the git repsitory." exit 1 @@ -62,6 +62,20 @@ for deb_tarball in "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT"*.debs.tar.gz; do tar xf "$deb_tarball" -C debs done +# +# Download the delphix upgrade verification debian package. +# Note, we always pull from the "master" build of the verification +# package, no matter what the UPSTREAM_BRANCH of the appliance is that +# we're building; this is intentional. +# + +AWS_S3_URI_UPGRADE_VERIFICATION=$(resolve_s3_uri \ + "$AWS_S3_URI_UPGRADE_VERIFICATION" \ + "$AWS_S3_PREFIX_UPGRADE_VERIFICATION" \ + "devops-gate/master/upgrade-verify/master/post-push/release/latest") + +download_delphix_s3_debs "$TOP/upgrade/debs" "$AWS_S3_URI_UPGRADE_VERIFICATION" + # # Generate an Aptly/APT repository # @@ -93,6 +107,18 @@ cp version.info.template version.info VERSION=$(dpkg -f "$(find debs/ -name 'delphix-entire-*' | head -n 1)" version) sed -i "s/@@VERSION@@/$VERSION/" version.info +# +# On 6.0 versions, the appliance stack expects to find the file +# verification-version.info pre-upgrade. This necessitates the +# packaging of this file in the upgrade image along with version.info. +# +cp verification-version.info.template verification-version.info + +# Include version information about this image. +VERIFICATION_VERSION=$(dpkg -f "$(find debs/ -name 'delphix-upgrade-verification*.deb' | head -n 1)" version) +sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" verification-version.info +sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" version.info + # # If we're running the build manually, and these variables are not # already set in the environment, we use an arbitrarily low falue to @@ -115,7 +141,7 @@ else sed -i "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/0.0.0.0/" version.info fi -sha256sum payload.tar.gz version.info prepare >SHA256SUMS +sha256sum payload.tar.gz version.info verification-version.info prepare >SHA256SUMS # # As a precaution, we disable "xtrace" so that we avoid exposing the @@ -150,6 +176,7 @@ tar -cf "$APPLIANCE_VARIANT.upgrade.tar" \ $(ls SHA256SUMS.sig.* 2>/dev/null) \ SHA256SUMS \ version.info \ + verification-version.info \ prepare \ payload.tar.gz diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 00000000..0a31c4e1 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TOP=$(git rev-parse --show-toplevel 2>/dev/null) + +function resolve_s3_uri() { + local pkg_uri="$1" + local pkg_prefix="$2" + local latest_subprefix="$3" + + local bucket="snapshot-de-images" + local jenkinsid="jenkins-ops" + local resolved_uri + + if [[ -n "$pkg_uri" ]]; then + resolved_uri="$pkg_uri" + elif [[ "$pkg_prefix" == s3* ]]; then + resolved_uri="$pkg_prefix" + elif [[ -n "$pkg_prefix" ]]; then + resolved_uri="s3://$bucket/$pkg_prefix" + elif [[ -n "$latest_subprefix" ]]; then + aws s3 cp --quiet \ + "s3://$bucket/builds/$jenkinsid/$latest_subprefix" . + resolved_uri="s3://$bucket/$(cat latest)" + rm -f latest + else + echo "Invalid arguments provided to resolve_s3_uri()" 2>&1 + exit 1 + fi + + if aws s3 ls "$resolved_uri" &>/dev/null; then + echo "$resolved_uri" + else + echo "'$resolved_uri' not found." 1>&2 + exit 1 + fi +} + +function download_delphix_s3_debs() { + local pkg_directory="$1" + local S3_URI="$2" + local tmp_directory + + tmp_directory=$(mktemp -d -p "$TOP/build" tmp.s3-debs.XXXXXXXXXX) + pushd "$tmp_directory" &>/dev/null + + aws s3 sync --only-show-errors "$S3_URI" . + sha256sum -c --strict SHA256SUMS + + mv ./*deb "$pkg_directory/" + + popd &>/dev/null + rm -rf "$tmp_directory" +} diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index a6f01996..8a50f4bf 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -289,6 +289,20 @@ function set_upgrade_property() { die "failed to read properties file after setting '$1=$2'" } +function apt_get() { + DEBIAN_FRONTEND=noninteractive apt-get \ + -o Dpkg::Options::="--force-confdef" \ + -o Dpkg::Options::="--force-confold" \ + "$@" +} + +function xargs_apt_get() { + DEBIAN_FRONTEND=noninteractive xargs apt-get \ + -o Dpkg::Options::="--force-confdef" \ + -o Dpkg::Options::="--force-confold" \ + "$@" +} + function verify_upgrade_not_in_progress() { . "$UPDATE_DIR/upgrade.properties" diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 235b314d..350ee509 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -28,20 +28,6 @@ function usage() { exit 2 } -function apt_get() { - DEBIAN_FRONTEND=noninteractive apt-get \ - -o Dpkg::Options::="--force-confdef" \ - -o Dpkg::Options::="--force-confold" \ - "$@" -} - -function xargs_apt_get() { - DEBIAN_FRONTEND=noninteractive xargs apt-get \ - -o Dpkg::Options::="--force-confdef" \ - -o Dpkg::Options::="--force-confold" \ - "$@" -} - while getopts ':rlBfsp:' c; do case $c in r | l | B | f | s) ;; # LX-72: For now, silently ignore these. diff --git a/upgrade/upgrade-scripts/verify-jar b/upgrade/upgrade-scripts/verify-jar index 2705acc8..7faa4b8b 100755 --- a/upgrade/upgrade-scripts/verify-jar +++ b/upgrade/upgrade-scripts/verify-jar @@ -20,6 +20,8 @@ IMAGE_VERSION=$(get_image_version) [[ -n "$IMAGE_VERSION" ]] || die "failed to determine image version" +IMAGE_PATH=$(get_image_path) +[[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" function verify_jar_verify_cleanup() { local rc="$?" @@ -42,6 +44,11 @@ function verify_jar_verify_cleanup() { die "failed to cleanup postgres for snapshot '${MDS_SNAPNAME}'." fi + if dpkg-query -W delphix-upgrade-verification &>/dev/null; then + apt_get remove --purge -y delphix-upgrade-verification || + die "failed to uninstall delphix-upgrade-verification package" + fi + [[ $rc -eq 0 ]] && # On success, we must report 100 progress. Java stack treats # script execution a failure if non-zero status is reported or @@ -142,10 +149,26 @@ report_progress 10 "Started application upgrade verification" [[ "$EUID" -ne 0 ]] && die "must be run as root" +# +# Install the delphix-upgrade-verification debian package from the +# unpack directory. This package is not installed as part of +# the execute script since no other package has a dependency on +# the delphix-upgrade-verification package. This is by design. +# This package is only meant to be used during upgrade verification +# and will be un-installed once verification is complete. This allows +# two engines that might have used different version of the +# delphix-upgrade-verification package to still have identical software +# post-upgrade. +# + +find "$IMAGE_PATH" -name "delphix-upgrade-verification*.deb" | + xargs_apt_get install -y --allow-downgrades || + die "failed to install delphix-upgrade-verification package" + JAVA_PARAMETERS=( "-Dlog.dir=/var/delphix/server/upgrade-verify" "-Dmdsverify=true" - "-jar" "/opt/delphix/server/lib/exec/upgrade-verify/upgrade-verify.jar" + "-jar" "/opt/delphix-upgrade-verification/upgrade-verify.jar" ) if [[ -n "$DLPX_DEBUG" ]] && $DLPX_DEBUG; then diff --git a/upgrade/verification-version.info.template b/upgrade/verification-version.info.template new file mode 100644 index 00000000..b128ecc4 --- /dev/null +++ b/upgrade/verification-version.info.template @@ -0,0 +1,26 @@ +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This file is consumed by the upgrade logic prior to upgrade in order to +# communicate the verification package version via API and CLI for the upgrade +# being performed. +# + +# +# The version of the upgrade verification package contained in the upgrade image. +# +VERIFICATION_VERSION=@@VERIFICATION_VERSION@@ diff --git a/upgrade/version.info.template b/upgrade/version.info.template index 5946e4c0..4259287d 100644 --- a/upgrade/version.info.template +++ b/upgrade/version.info.template @@ -57,3 +57,8 @@ MINIMUM_VERSION=@@MINIMUM_VERSION@@ # MINIMUM_VERSION above. # MINIMUM_REBOOT_OPTIONAL_VERSION=@@MINIMUM_REBOOT_OPTIONAL_VERSION@@ + +# +# The version of the upgrade verification package contained in the upgrade image. +# +VERIFICATION_VERSION=@@VERIFICATION_VERSION@@ From edeebac3c841872847a38e1c2409f4d5ab2e755f Mon Sep 17 00:00:00 2001 From: sara hartse Date: Mon, 13 Apr 2020 14:42:31 -0700 Subject: [PATCH 034/197] correct branch.config --- branch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/branch.config b/branch.config index 9e572e67..8b13b1e8 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/stage" +UPSTREAM_BRANCH="6.0/release" From f5c954c7d84dd0f1d59b499d3ffefda7cc8964b8 Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Thu, 11 Jun 2020 12:13:14 -0400 Subject: [PATCH 035/197] copy static DNS configuration for not-in-place upgrades (#470) --- upgrade/upgrade-scripts/upgrade-container | 1 + 1 file changed, 1 insertion(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 540e4982..edc42393 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -628,6 +628,7 @@ function migrate_configuration() { /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key.pub /etc/sudoers.d/90-cloud-init-users + /etc/systemd/resolved.conf.d/delphix-static.conf /etc/zfs/zpool.cache EOF From a3ac7e6a773ba0b1ce7f45b3d356380d11b700ff Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 7 Jul 2020 14:12:04 -0700 Subject: [PATCH 036/197] Remove unnecessary files from upgrade payload (#472) (#473) Currently, when we generate an upgrade image, we include the entire contents of "~/.aptly" in the image. It turns out, this causes us to include two identical copies of each package, one under "~/.aptly/pool" and another under "~/.aptly/public/pool". For the upgrade image, we only need the contents of the "~/.aptly/public" directory, so this change modifies the upgrade image generation logic, as well as the upgrade image application logic, to take this into account. As a result, this significantly reduces the size of the upgrade image (roughly a 50% reduction in size). --- scripts/build-upgrade-image.sh | 2 +- upgrade/upgrade-scripts/execute | 2 +- upgrade/upgrade-scripts/upgrade-container | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build-upgrade-image.sh b/scripts/build-upgrade-image.sh index fdfbd002..e4bc5a94 100755 --- a/scripts/build-upgrade-image.sh +++ b/scripts/build-upgrade-image.sh @@ -99,7 +99,7 @@ aptly publish repo -skip-contents -skip-signing upgrade-repository # be realtively small (and that file's size grows linearly with the # number of files contained in the "upgrade image" tarball). # -tar -I pigz -cf "payload.tar.gz" -C upgrade-scripts . -C ~/.aptly . +tar -I pigz -cf "payload.tar.gz" -C upgrade-scripts . -C ~/.aptly/public . cp version.info.template version.info diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 350ee509..7461592d 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -115,7 +115,7 @@ if [[ -f /etc/apt/sources.list ]]; then fi cat </etc/apt/sources.list || -deb [trusted=yes] file://$IMAGE_PATH/public bionic delphix +deb [trusted=yes] file://$IMAGE_PATH bionic delphix EOF die "failed to configure apt sources" diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index edc42393..a8896a39 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -231,7 +231,7 @@ function create_upgrade_container() { # debootstrap --no-check-gpg \ --components=delphix --include=systemd-container \ - bionic "$DIRECTORY" "file://$IMAGE_PATH/public" 1>&2 || + bionic "$DIRECTORY" "file://$IMAGE_PATH" 1>&2 || die "failed to debootstrap upgrade filesystem" # From 6862a3920003350d3c14fc0128a9d01ad13b907d Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Wed, 22 Jul 2020 15:59:19 -0400 Subject: [PATCH 037/197] Checkout the master branch instead of projects/dx4linux on dcol. (#475) The projects/dx4linux branch in dcenter-gate is no longer used. All work is being done on master, and so that's the branch that needs to be checked out on deployed dcol instances. --- .../misc/ansible-roles/appliance-build.dcenter/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index a421ee50..84a0a6a6 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -46,7 +46,7 @@ - git: repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' - version: projects/dx4linux + version: master dest: /opt/dcenter/lib/dcenter-gate accept_hostkey: yes update: no From 3f0ae5499e219eb1baf6598ae6dc9c7d25cb5655 Mon Sep 17 00:00:00 2001 From: Don Brady Date: Wed, 22 Jul 2020 15:49:12 -0600 Subject: [PATCH 038/197] dcenter image should increase mountd thread count [Backport of TOOL-9772] (#477) --- .../appliance-build.dcenter/tasks/main.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 84a0a6a6..7db19951 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -1,5 +1,5 @@ # -# Copyright 2018 Delphix +# Copyright 2018,2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,4 +87,17 @@ regexp: '^preserve_hostname: false' line: 'preserve_hostname: true' +# +# The default setting for the number of nfs threads is too low. To +# improve performance we reset the value to 64 which mimics what +# we use on the delphix engine. +# +- lineinfile: + path: /etc/default/nfs-kernel-server + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: '^RPCNFSDCOUNT=', line: 'RPCNFSDCOUNT=64' } + - { regexp: '^RPCMOUNTDOPTS=', line: 'RPCMOUNTDOPTS="--num-threads=5 --manage-gids"' } + - command: systemctl disable bind9.service From 5387ad3e0955e2fb560af63b849de72df01bd31e Mon Sep 17 00:00:00 2001 From: George Wilson Date: Thu, 23 Jul 2020 10:49:16 -0500 Subject: [PATCH 039/197] backport tenacity requirement (part of TOOL-9418) (#478) --- .../misc/ansible-roles/appliance-build.dcenter/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 7db19951..5a5881eb 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -30,6 +30,7 @@ - python-paramiko - python-pip - python-requests + - python-tenacity - python2.7 - telnet state: present From f81f0704d88f52ffaae0bc31af11ffe6883dd127 Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Wed, 29 Jul 2020 19:20:17 -0400 Subject: [PATCH 040/197] DLPX-71245 [Backport of DLPX-71244 to 6.0.3.0] fix for DLPX-69049 needs to also update grub for migrations (#480) --- live-build/misc/migration-scripts/dx_apply | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 75efc62b..7889f6cb 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -275,6 +275,7 @@ LX_CMDLINE=( 'zfsforce=1' 'mitigations=off' 'elevator=noop' + 'init_on_alloc=0' ) # From 78a79852b4d7ce7dff11c105574362a7a3f8c033 Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Mon, 3 Aug 2020 15:30:44 -0400 Subject: [PATCH 041/197] DLPX-70914 [Backport of DLPX-70835 to 6.0.4.0] disable usb-storage device module loading in delphix 6.x (5.3 migration part) (#483) --- live-build/misc/migration-scripts/dx_apply | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply index 7889f6cb..c02a9df9 100755 --- a/live-build/misc/migration-scripts/dx_apply +++ b/live-build/misc/migration-scripts/dx_apply @@ -276,6 +276,7 @@ LX_CMDLINE=( 'mitigations=off' 'elevator=noop' 'init_on_alloc=0' + 'usbcore.nousb=1' ) # From cde27db0475940e8a0037ee82242794e8ac35fa1 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Tue, 4 Aug 2020 22:40:25 -0700 Subject: [PATCH 042/197] DLPX-71311 [Backport of DLPX-69447 to 6.0.4.0] Fixed hotfixes should be removed from /etc/hotfix during upgrade application (#484) --- upgrade/upgrade-scripts/common.sh | 1 + upgrade/upgrade-scripts/upgrade | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 8a50f4bf..36bad5f2 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -18,6 +18,7 @@ # shellcheck disable=SC2034 UPDATE_DIR="/var/dlpx-update" LOG_DIRECTORY="/var/tmp/delphix-upgrade" +HOTFIX_PATH="/etc/hotfix" # # We embed information as dataset properties in our rootfs containers. diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 7c88cb17..92513dd0 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -446,6 +446,10 @@ function finalize() { rm -rf "$IMAGE_PATH" || die "failed to remove unpacked upgrade image" remove_upgrade_properties + + if [[ -f "$HOTFIX_PATH" ]]; then + rm -f "$HOTFIX_PATH" || die "failed to remove hotfix file" + fi } [[ "$EUID" -ne 0 ]] && die "must be run as root" From e8e31748fecd124b8029fef0efcb307b5160a015 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Wed, 12 Aug 2020 03:55:05 -0700 Subject: [PATCH 043/197] Backport: Install recovery env (#487) --- .../meta/main.yml | 19 +++++++++++++++++ .../tasks/main.yml | 21 +++++++++++++++++++ .../external-standard/ansible/playbook.yml | 1 + .../internal-dev/ansible/playbook.yml | 1 + .../variants/internal-qa/ansible/playbook.yml | 1 + 5 files changed, 43 insertions(+) create mode 100644 live-build/misc/ansible-roles/appliance-build.recovery-environment/meta/main.yml create mode 100644 live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml diff --git a/live-build/misc/ansible-roles/appliance-build.recovery-environment/meta/main.yml b/live-build/misc/ansible-roles/appliance-build.recovery-environment/meta/main.yml new file mode 100644 index 00000000..cc711886 --- /dev/null +++ b/live-build/misc/ansible-roles/appliance-build.recovery-environment/meta/main.yml @@ -0,0 +1,19 @@ +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +dependencies: + - role: appliance-build.virtualization-common diff --git a/live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml new file mode 100644 index 00000000..a701997d --- /dev/null +++ b/live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml @@ -0,0 +1,21 @@ +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +- apt: + name: recovery-environment + state: present + diff --git a/live-build/variants/external-standard/ansible/playbook.yml b/live-build/variants/external-standard/ansible/playbook.yml index 32d2dae7..9402d53f 100644 --- a/live-build/variants/external-standard/ansible/playbook.yml +++ b/live-build/variants/external-standard/ansible/playbook.yml @@ -23,3 +23,4 @@ - appliance-build.minimal-common - appliance-build.masking-common - appliance-build.virtualization-common + - appliance-build.recovery-environment diff --git a/live-build/variants/internal-dev/ansible/playbook.yml b/live-build/variants/internal-dev/ansible/playbook.yml index bac20acb..c2ecff0d 100644 --- a/live-build/variants/internal-dev/ansible/playbook.yml +++ b/live-build/variants/internal-dev/ansible/playbook.yml @@ -41,3 +41,4 @@ - appliance-build.virtualization-common - appliance-build.virtualization-development - appliance-build.zfsonlinux-development + - appliance-build.recovery-environment diff --git a/live-build/variants/internal-qa/ansible/playbook.yml b/live-build/variants/internal-qa/ansible/playbook.yml index 21df7ffc..88aee943 100644 --- a/live-build/variants/internal-qa/ansible/playbook.yml +++ b/live-build/variants/internal-qa/ansible/playbook.yml @@ -25,3 +25,4 @@ - appliance-build.masking-common - appliance-build.qa-internal - appliance-build.virtualization-common + - appliance-build.recovery-environment From ee33ea633ae527d1ed2931cb0163a3f17b23bba2 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 17 Aug 2020 14:48:12 -0700 Subject: [PATCH 044/197] Fix calls to "verify_upgrade_not_in_progress" (#462) (#486) The "verify_upgrade_not_in_progress" function only works correctly if the "UPGRADE_TYPE" variable is not set prior to the function being called. Unfortunately, in the two places we currently call that function, that variable will have already been set. This change fixes the issue by moving the calls to the function, such that we call it prior to setting the variable. Additionally, we add a check to the function to catch this error in usage earlier, and emit a more appropriate error message. Closes #461 --- upgrade/upgrade-scripts/common.sh | 9 ++++++++- upgrade/upgrade-scripts/upgrade | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 36bad5f2..9598cafe 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -305,7 +305,14 @@ function xargs_apt_get() { } function verify_upgrade_not_in_progress() { - . "$UPDATE_DIR/upgrade.properties" + # + # This function only works properly if the UPGRADE_TYPE variable + # is not set prior to this function being called. Thus, to help + # catch cases where this function is called incorrectly, we + # verify the variable is empty before proceeding. + # + [[ -z "$UPGRADE_TYPE" ]] || die "UPGRADE_TYPE already set" + . "$UPDATE_DIR/upgrade.properties" &>/dev/null [[ -z "$UPGRADE_TYPE" ]] || die "upgrade currently in-progress" } diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 92513dd0..04e6caf2 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -471,18 +471,20 @@ shift $((OPTIND - 1)) case "$1" in deferred) + verify_upgrade_not_in_progress + UPGRADE_TYPE="DEFERRED" shift 1 verify_upgrade_is_allowed verify_upgrade_in_place_is_allowed - verify_upgrade_not_in_progress upgrade_in_place "$@" ;; full) + verify_upgrade_not_in_progress + UPGRADE_TYPE="FULL" shift 1 verify_upgrade_is_allowed - verify_upgrade_not_in_progress # # FULL upgrade always perform a reboot but can take on two From 9a6759ce1b5b789351f638fd1aa03bd5aedcd7a7 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 17 Aug 2020 14:48:27 -0700 Subject: [PATCH 045/197] Fix "upgrade-container stop" return code on success (#433) (#434) This change fixes a regression introduced in acb60f4c. Without this change, calling "upgrade-container stop" will return an error code, even when the call successfully stops the container. With this change, when the container is stopped successfully, we'll now properly return 0. --- upgrade/upgrade-scripts/upgrade-container | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index a8896a39..a18ed7d4 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -420,6 +420,13 @@ function stop() { machinectl status "$CONTAINER" &>/dev/null && die "timeout waiting for container termination: '$CONTAINER'" + + # + # We don't want to return the error code from the call to + # "machinectl status" above, so we explicitly return success if + # we've reached this far. + # + return 0 } function destroy() { From 7d6c2f5556fba0408f6879fdfbf2cf932555f851 Mon Sep 17 00:00:00 2001 From: Raul Santelices Date: Tue, 18 Aug 2020 14:01:21 -0400 Subject: [PATCH 046/197] fix branch.config --- branch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/branch.config b/branch.config index 9e572e67..8b13b1e8 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/stage" +UPSTREAM_BRANCH="6.0/release" From c099300ca5decf660eb5fa1c313dcd33430b0074 Mon Sep 17 00:00:00 2001 From: Raul Santelices Date: Thu, 17 Sep 2020 12:26:00 -0400 Subject: [PATCH 047/197] fix branch.config --- branch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/branch.config b/branch.config index 8b13b1e8..4a3b6bf9 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/release" +UPSTREAM_BRANCH="6.0/patch" From 0d7f4b5dbcecbaeac8bd8944ae974f4947dc204d Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Fri, 16 Oct 2020 14:42:52 -0700 Subject: [PATCH 048/197] Backports of Python package changes (#492) * Add py-six to DCenter appliance-build ansible role (#485) Co-authored-by: brandon.lim * Update DCenter Python package list (#488) - Add python-dbg and python-dev for use with GDB. - Remove python-requests, since we no longer depend on it as of TOOL-9913. - Add python-pyvmomi, because we eventually plan to migrate from PySphere to pyVmomi. Installing the package now enables us to begin preliminary testing with pyVmomi. - Add python-virtualenv, because we eventually plan to deploy our dependencies in a virtual environment. Installing the package now enables us to begin preliminary testing of virtual environments. - Add Python 3 versions of all Python 2 packages, because we eventually plan to migrate from Python 2 to Python 3. Installing Python 3 now enables us to begin preliminary testing with Python 3. Co-authored-by: brandon.lim --- .../appliance-build.dcenter/tasks/main.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 5a5881eb..b8011b72 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -26,12 +26,26 @@ - libldap2-dev - libsasl2-dev - nfs-kernel-server + - python-dbg + - python-dev - python-ldap - python-paramiko - python-pip - - python-requests + - python-pyvmomi + - python-six - python-tenacity + - python-virtualenv - python2.7 + - python3 + - python3-dbg + - python3-dev + - python3-ldap + - python3-paramiko + - python3-pip + - python3-pyvmomi + - python3-six + - python3-tenacity + - python3-virtualenv - telnet state: present register: result From 9a2273a2754b052d1bd0fa149189a37a1de24c29 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 2 Dec 2020 09:13:05 -0500 Subject: [PATCH 049/197] TOOL-10595 [Backport of TOOL-10160 to 6.0.6.0] linux-pkg rework: main appliance-build change (#496) --- live-build/build.gradle | 9 +------ scripts/build-ancillary-repository.sh | 39 +++++++-------------------- scripts/common.sh | 21 +++++++++++++++ 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/live-build/build.gradle b/live-build/build.gradle index c9460587..c00e808f 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -21,14 +21,7 @@ apply from: "${rootProject.projectDir}/gradle-lib/util.gradle" task ancillaryRepository(type: Exec) { inputs.file "${rootProject.projectDir}/scripts/build-ancillary-repository.sh" - for (envVar in ["AWS_S3_URI_VIRTUALIZATION", - "AWS_S3_URI_USERLAND_PKGS", - "AWS_S3_URI_MASKING", - "AWS_S3_URI_ZFS", - "AWS_S3_PREFIX_VIRTUALIZATION", - "AWS_S3_PREFIX_MASKING", - "AWS_S3_PREFIX_USERLAND_PKGS", - "AWS_S3_PREFIX_KERNEL_PKGS"]) { + for (envVar in ["COMBINED_PACKAGES_S3_URL"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) } diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index d487a06a..8ceeb7ce 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -62,12 +62,12 @@ function build_ancillary_repository() { } # -# The first-party packages produced by Delphix are stored in Amazon S3. +# The packages produced by Delphix are stored in Amazon S3. # Thus, in order to populate the ancillary repository with these # packages, they must be downloaded from S3, so they can be then # inserted into the Aptly repository. # -# Here, we determine the URI of each of the first-party packages, and +# Here, we determine the URI of each of the Delphix packages, and # then use these URIs to download the packages later. Making this # determination is a little complex, and is dependent on the policy set # forth by the teams producing and storing the packages. @@ -76,12 +76,12 @@ function build_ancillary_repository() { # which the packages are downloaded: # # 1. If the package specific AWS_S3_URI environment variable is provided -# (e.g. AWS_S3_URI_VIRTUALIZATION), then this URI will be used to +# (e.g. AWS_S3_URI_UPGRADE_VERIFICATION), then this URI will be used to # download the package. This is the simplest case, and enables the # user of this script to directly influence this script. # # 2. If the package specific AWS_S3_PREFIX environment variable is -# provided (e.g. AWS_S3_PREFIX_VIRTUALIZATION), then this value is +# provided (e.g. AWS_S3_PREFIX_UPGRADE_VERIFICATION), then this value is # used to build the URI that will be used based on the default S3 # bucket that is used. # @@ -115,25 +115,9 @@ else fi echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" -AWS_S3_URI_VIRTUALIZATION=$(resolve_s3_uri \ - "$AWS_S3_URI_VIRTUALIZATION" \ - "$AWS_S3_PREFIX_VIRTUALIZATION" \ - "dlpx-app-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") - -AWS_S3_URI_MASKING=$(resolve_s3_uri \ - "$AWS_S3_URI_MASKING" \ - "$AWS_S3_PREFIX_MASKING" \ - "dms-core-gate/${UPSTREAM_BRANCH}/build-package/post-push/latest") - -AWS_S3_URI_USERLAND_PKGS=$(resolve_s3_uri \ - "$AWS_S3_URI_USERLAND_PKGS" \ - "$AWS_S3_PREFIX_USERLAND_PKGS" \ - "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/userland/post-push/latest") - -AWS_S3_URI_KERNEL_PKGS=$(resolve_s3_uri \ - "$AWS_S3_URI_KERNEL_PKGS" \ - "$AWS_S3_PREFIX_KERNEL_PKGS" \ - "devops-gate/master/linux-pkg-build/${UPSTREAM_BRANCH}/kernel/post-push/latest") +AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ + "$AWS_S3_URI_COMBINED_PACKAGES" "" \ + "devops-gate/master/linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") # # All package files will be placed into this temporary directory, such @@ -144,13 +128,10 @@ mkdir -p "$TOP/build" PKG_DIRECTORY=$(mktemp -d -p "$TOP/build" tmp.pkgs.XXXXXXXXXX) # -# Now that we've determined the URI of all first-party packages, we can -# proceed to download these packages. +# Now that we've determined the URI of the Delphix-built packages, we can +# download them. # -download_delphix_s3_debs "$PKG_DIRECTORY" "$AWS_S3_URI_VIRTUALIZATION" -download_delphix_s3_debs "$PKG_DIRECTORY" "$AWS_S3_URI_MASKING" -download_delphix_s3_debs "$PKG_DIRECTORY" "$AWS_S3_URI_USERLAND_PKGS" -download_delphix_s3_debs "$PKG_DIRECTORY" "$AWS_S3_URI_KERNEL_PKGS" +download_delphix_s3_debs_multidir "$PKG_DIRECTORY" "$AWS_S3_URI_COMBINED_PACKAGES/packages" # # Now that our temporary package directory has been populated with all diff --git a/scripts/common.sh b/scripts/common.sh index 0a31c4e1..ef7dd7f5 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -66,3 +66,24 @@ function download_delphix_s3_debs() { popd &>/dev/null rm -rf "$tmp_directory" } + +function download_delphix_s3_debs_multidir() { + local pkg_directory="$1" + local S3_URI="$2" + local tmp_directory + + tmp_directory=$(mktemp -d -p "$TOP/build" tmp.s3-debs.XXXXXXXXXX) + pushd "$tmp_directory" &>/dev/null + + aws s3 sync --only-show-errors "$S3_URI" . + + for subdir in */; do + pushd "$subdir" &>/dev/null + sha256sum -c --strict SHA256SUMS + mv ./*deb "$pkg_directory/" + popd &>/dev/null + done + + popd &>/dev/null + rm -rf "$tmp_directory" +} From 1b61e86407aae44868a2e5891ebd5b5eac13a30a Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 4 Dec 2020 07:14:27 -0800 Subject: [PATCH 050/197] Github Action "::set-env" is no longer supported (#501) (#502) The Github Action "::set-env" command is no longer supported and results in failures when it is used. This change updates our scripts to remove the usage of that command, and use the "GITHUB_ENV" interface instead. Closes #495 --- .github/scripts/install-ansible-lint.sh | 10 +++++----- .github/workflows/main.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/install-ansible-lint.sh b/.github/scripts/install-ansible-lint.sh index 53b73b39..3ac7e126 100755 --- a/.github/scripts/install-ansible-lint.sh +++ b/.github/scripts/install-ansible-lint.sh @@ -6,10 +6,10 @@ git checkout v3.4.21 git branch -D master # -# GitHub Actions exposes some "debugging commands" that can be used to +# GitHub Actions exposes the GITHUB_ENV file that can be used to # manipulate the environment of the job that's running. In this case, we -# use the "set-env" command to modify the environment of the job, to -# edit the PATH and PYTHONPATH global variables. +# use it to modify the environment of the job, to edit the PATH and +# PYTHONPATH global variables. # -echo "::set-env name=PATH::${PATH}:/opt/ansible-lint/bin" -echo "::set-env name=PYTHONPATH::${PYTHONPATH}:/opt/ansible-lint/lib" +echo "PATH=${PATH}:/opt/ansible-lint/bin" >> ${GITHUB_ENV} +echo "PYTHONPATH=${PYTHONPATH}:/opt/ansible-lint/lib" >> ${GITHUB_ENV} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c27a55f4..640b0f11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v1 - run: sudo ./.github/scripts/install-gradle.sh - - run: sudo ./.github/scripts/install-ansible-lint.sh + - run: sudo -E ./.github/scripts/install-ansible-lint.sh - run: /opt/gradle-5.1/bin/gradle ansibleCheck check-shellcheck: runs-on: ubuntu-18.04 From c26b447492dd39d87234f1dd2f9f8b49f3786874 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 4 Dec 2020 13:31:43 -0500 Subject: [PATCH 051/197] DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install (#499) --- .../roles/appliance-build.bootstrap/tasks/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 9b067a02..4b71d30d 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -44,6 +44,20 @@ - zfsutils-linux state: present +# +# See DLPX-72860 for more info on the custom package. +# +- name: Custom livecd-rootfs package | Download + get_url: + url: 'https://artifactory.delphix.com:443/artifactory/linux-pkg/livecd-rootfs/6.0.6.0/livecd-rootfs_2.525.47-delphix1_amd64.deb' + dest: '/var/tmp/livecd-rootfs_2.525.47-delphix1_amd64.deb' + checksum: 'sha256:9f090adf288d115b2eb10d2dced2a76113339eb95dc5db91fac4b89b2bef07a0' + +- name: Custom livecd-rootfs package | Install + apt: + deb: '/var/tmp/livecd-rootfs_2.525.47-delphix1_amd64.deb' + state: present + - modprobe: name: zfs state: present From 8004a11a229f93abc8c3666dcb1193d4a2056b4d Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Fri, 4 Dec 2020 16:06:44 -0800 Subject: [PATCH 052/197] DLPX-73083 [Backport of DLPX-72429 to 6.0.6.0] ssh to admin CLI shows error message after force-not-in-place upgrade, permissions issue on /export dir (#504) --- upgrade/upgrade-scripts/common.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 9598cafe..7640b41f 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -20,6 +20,14 @@ UPDATE_DIR="/var/dlpx-update" LOG_DIRECTORY="/var/tmp/delphix-upgrade" HOTFIX_PATH="/etc/hotfix" +# +# The virtualization service uses a different umask than the default. Thus to account for the +# fact that these scripts may be running with a non-default umask, we explicitly change it back +# to the default value here. This helps ensure any files and directories generated by these scripts, +# will be created with the correct permissions, regardless of the umask of the calling process. +# +umask 0022 + # # We embed information as dataset properties in our rootfs containers. # These are the names of these properties. From 66c4de648e9ccd33311666cda8d614b3314e06d5 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 7 Dec 2020 09:22:21 -0800 Subject: [PATCH 053/197] DLPX-72721 Cleanup old Illumos datasets on Linux to Linux upgrade (#500) (#505) --- upgrade/upgrade-scripts/upgrade | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 04e6caf2..8069100a 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -389,6 +389,19 @@ function finalize() { ;; esac + # + # If we've reached this point, it means we're finalizing an + # ugprade from a linux-based release, to another linux-based + # release. Thus, we necessarily must be at least 2 versions + # beyond any illumos-based release, if this system started from + # an illumos-based release. Thus, at this point, we can safely + # perform the final cleanup of any illumos-based datasets. + # + if [[ -f /var/delphix/migration/final-illumos-cleanup-required ]]; then + /opt/delphix/server/bin/upgrade/illumos-cleanup final || + die "failed to clean up illumos datasets" + fi + # # This first pass checks for available "execute.upgrade" snapshots # and deletes snapshots that are older than the base upgrade version. From 76ad37cd3a0726574d74453f4335d0c9c3c9761a Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Mon, 14 Dec 2020 13:01:39 -0800 Subject: [PATCH 054/197] Backports of DCenter package updates (#508) Amends #488. Updates the DCenter package list to improve debuggability, adapt to recent changes in dcenter-gate, and prepare for future work. - Adding debug symbols via nfs-common-dbgsym and nfs-kernel-server-dbgsym to improve debuggability. - Removing python3-paramiko, as it is no longer needed when running dcenter-gate with pyVmomi. Note that Paramiko must remain installed for Python 2 until we drop support for PySphere (scheduled for later this month). - Removing virtualenv in favor of venv as in QI-1699. Note that we do not currently use a virtual environment, but we plan to start using one in future packaging and deployment work. We will likely be Python 3 only by the time that work is begun, so it makes more sense to use the native Python 3 venv than virtualenv. - Adding targetcli-fb in order to support prototyping dc volume. Also removes dangling references to dcenter.conf. --- .../appliance-build.dcenter/tasks/main.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index b8011b72..24009464 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -25,7 +25,9 @@ - isc-dhcp-server - libldap2-dev - libsasl2-dev + - nfs-common-dbgsym - nfs-kernel-server + - nfs-kernel-server-dbgsym - python-dbg - python-dev - python-ldap @@ -34,18 +36,17 @@ - python-pyvmomi - python-six - python-tenacity - - python-virtualenv - python2.7 - python3 - python3-dbg - python3-dev - python3-ldap - - python3-paramiko - python3-pip - python3-pyvmomi - python3-six - python3-tenacity - - python3-virtualenv + - python3-venv + - targetcli-fb - telnet state: present register: result @@ -66,11 +67,6 @@ accept_hostkey: yes update: no -- copy: - dest: "/etc/dcenter.conf" - content: | - [[ -z "$DC_ROOT" ]] && export DC_ROOT="dcenter" - # # By default, ubuntu restricts directories where dhcpd and named # can operate. For dcenter, we maintain the dhcp configuration From 02587ac404667f1046527a66d286d8dcda29436d Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 16 Dec 2020 11:08:19 -0500 Subject: [PATCH 055/197] Revert "DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install" (#512) This reverts commit c26b447492dd39d87234f1dd2f9f8b49f3786874. --- .../roles/appliance-build.bootstrap/tasks/main.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 4b71d30d..9b067a02 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -44,20 +44,6 @@ - zfsutils-linux state: present -# -# See DLPX-72860 for more info on the custom package. -# -- name: Custom livecd-rootfs package | Download - get_url: - url: 'https://artifactory.delphix.com:443/artifactory/linux-pkg/livecd-rootfs/6.0.6.0/livecd-rootfs_2.525.47-delphix1_amd64.deb' - dest: '/var/tmp/livecd-rootfs_2.525.47-delphix1_amd64.deb' - checksum: 'sha256:9f090adf288d115b2eb10d2dced2a76113339eb95dc5db91fac4b89b2bef07a0' - -- name: Custom livecd-rootfs package | Install - apt: - deb: '/var/tmp/livecd-rootfs_2.525.47-delphix1_amd64.deb' - state: present - - modprobe: name: zfs state: present From b5d53e8d194df2b9e44bd9689c7dab830f5da867 Mon Sep 17 00:00:00 2001 From: mneetha Date: Thu, 17 Dec 2020 10:01:53 -0800 Subject: [PATCH 056/197] QA-25826 [Backport of QA-20893] Add the snmptrapd pkg to the delphix VM (#513) Co-authored-by: Neetha Menon --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 517f5a80..4de3b32a 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -15,10 +15,13 @@ # # This file is intended only for QA-specific testing items and frameworks. # Anything required by customers should not be added here. +# snmptrapd pkg is added to test snmp traps feature. # - apt: - name: nftables + name: + - nftables + - snmptrapd state: present register: result until: result is not failed From c74662f75b3ec76937bfd93f35cc04cc5d045fc8 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Thu, 17 Dec 2020 14:24:37 -0800 Subject: [PATCH 057/197] Backport hot-add to 6.0.6.0 (#516) --- live-build/config/hooks/vm-artifacts/template.ovf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/template.ovf b/live-build/config/hooks/vm-artifacts/template.ovf index 9d1bf8bd..78edfb51 100644 --- a/live-build/config/hooks/vm-artifacts/template.ovf +++ b/live-build/config/hooks/vm-artifacts/template.ovf @@ -130,12 +130,12 @@ 10 - + - + From 4c0cc2fed668077990529f9e7b02c21b182fca76 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 17 Dec 2020 22:15:39 -0500 Subject: [PATCH 058/197] [Backport to 6.0.6.0] Enable generation of "sparse" upgrade image (#514) This change adds a new script that can be used to generate a "sparse" upgrade image. For example, given two input upgrade images, we can generate a sparse image like so: $ ./scripts/aptly-repo-from-image-diff.sh ../master/internal-dev.upgrade.tar ../6.0-release/internal-dev.upgrade.tar $ ./scripts/upgrade-image-from-aptly-repo.sh sparse-internal-dev $ ls -l build/artifacts/sparse-internal-dev.upgrade.tar -rw-r--r-- 1 delphix staff 5459210240 Jul 31 19:47 build/artifacts/sparse-internal-dev.upgrade.tar Additionally, we can inspect this new artifact and verify the set of packages it contains: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 199 And contrast this with the number of packages contained in both the "master" and "6.0-release" images: - master: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1042 - 6.0/release: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1080 Co-authored-by: Prakash Surya --- .gitignore | 4 +- scripts/aptly-repo-from-debs.sh | 82 ++++++++++ scripts/aptly-repo-from-image-diff.sh | 117 +++++++++++++++ scripts/build-upgrade-image.sh | 177 ++-------------------- scripts/common.sh | 5 + scripts/upgrade-image-from-aptly-repo.sh | 183 +++++++++++++++++++++++ 6 files changed, 404 insertions(+), 164 deletions(-) create mode 100755 scripts/aptly-repo-from-debs.sh create mode 100755 scripts/aptly-repo-from-image-diff.sh create mode 100755 scripts/upgrade-image-from-aptly-repo.sh diff --git a/.gitignore b/.gitignore index 64caed74..a3f90ab0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ # limitations under the License. # +*.swp +*.orig .gradle/ .gradleUserHome/ build/ @@ -22,4 +24,4 @@ build/ *.iml # Ignore the .idea directory that IntelliJ puts in the root of projects -.idea \ No newline at end of file +.idea diff --git a/scripts/aptly-repo-from-debs.sh b/scripts/aptly-repo-from-debs.sh new file mode 100755 index 00000000..5874417b --- /dev/null +++ b/scripts/aptly-repo-from-debs.sh @@ -0,0 +1,82 @@ +#!/bin/bash -ex +# +# Copyright 2018-2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is responsible for generating a new Aptly repository by +# taking and combining all of the deb.tar.gz tarballs produced by live +# build for this variant (of which there will be one per supported +# platform). The generated repository will be stored at ~/.aptly/public. +# This new repository can then be used to generate a new upgrade image, +# by running the "upgrade-image-from-aptly-repo.sh" script. +# + +. "${BASH_SOURCE%/*}/common.sh" + +if [[ -z "$TOP" ]]; then + echo "Must be run inside the git repsitory." + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "Must specify a single variant." 1>&2 + exit 1 +fi + +cd "$TOP/upgrade" +APPLIANCE_VARIANT=$1 + +rm -rf ~/.aptly +rm -rf debs +mkdir debs + +# +# For upgrade images that we ship to customers, we will need to include +# the packages for every platform that we support. Building for every +# platform can be time-comsuming though, so for developer convenience, +# here we just take the artifacts from the live-build stage for whatever +# appliance versions were built (making sure that we built for at least +# one platform), and build an upgrade image from that. +# +LIVE_BUILD_OUTPUT_DIR="$TOP/live-build/build/artifacts" +if ! compgen -G "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT*.debs.tar.gz"; then + echo "No live-build artifacts found for this variant" >&2 + exit 1 +fi +for deb_tarball in "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT"*.debs.tar.gz; do + tar xf "$deb_tarball" -C debs +done + +# +# Download the delphix upgrade verification debian package. +# Note, we always pull from the "master" build of the verification +# package, no matter what the UPSTREAM_BRANCH of the appliance is that +# we're building; this is intentional. +# + +AWS_S3_URI_UPGRADE_VERIFICATION=$(resolve_s3_uri \ + "$AWS_S3_URI_UPGRADE_VERIFICATION" \ + "$AWS_S3_PREFIX_UPGRADE_VERIFICATION" \ + "devops-gate/master/upgrade-verify/master/post-push/release/latest") + +download_delphix_s3_debs "$TOP/upgrade/debs" "$AWS_S3_URI_UPGRADE_VERIFICATION" + +# +# Generate an Aptly/APT repository +# +aptly repo create -distribution=bionic -component=delphix upgrade-repository +aptly repo add upgrade-repository debs +aptly publish repo -skip-contents -skip-signing upgrade-repository diff --git a/scripts/aptly-repo-from-image-diff.sh b/scripts/aptly-repo-from-image-diff.sh new file mode 100755 index 00000000..826d503c --- /dev/null +++ b/scripts/aptly-repo-from-image-diff.sh @@ -0,0 +1,117 @@ +#!/bin/bash -x +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is responsible for taking two upgrade images as input, and +# generating a new Aptly repository containing the set difference of +# these two images (i.e. A - B), which it stores at ~/.aptly/public. +# This new repository can then be used to generate a new upgrade image, +# by running the "upgrade-image-from-aptly-repo.sh" script. +# + +. "${BASH_SOURCE%/*}/common.sh" + +set -o pipefail + +function cleanup() { + [[ -n "$UNPACK_DIR" ]] && [[ -d "$UNPACK_DIR" ]] && rm -rf "$UNPACK_DIR" +} + +function usage() { + echo "$(basename "$0"): $*" >&2 + echo "Usage: $(basename "$0") " + exit 2 +} + +function import_image_into_aptly() { + local reponame="$1" + local imagepath="$2" + + mkdir "$reponame" || die "'mkdir $reponame' failed" + pushd "$reponame" &>/dev/null || die "'pushd $reponame' failed" + + tar -xf "$imagepath" || die "failed to extract image '$imagepath'" + tar -xf payload.tar.gz || die "failed to extract payload" + + aptly repo create "$reponame" || + die "failed to create repository: '$reponame'" + aptly repo add "$reponame" pool || + die "failed to add packages to repository: '$reponame'" + + popd &>/dev/null || die "'popd' failed" + rm -rf "$reponame" || die "'rm -rf $reponame' failed" +} + +trap cleanup EXIT + +[[ $# -gt 2 ]] && usage "too many arguments specified" +[[ $# -lt 2 ]] && usage "too few arguments specified" + +IMAGE_A_PATH=$(readlink -f "$1") +[[ -n "$IMAGE_A_PATH" ]] || die "unable to determine image A path" +[[ -f "$IMAGE_A_PATH" ]] || die "image path is not a file: '$IMAGE_A_PATH'" + +IMAGE_B_PATH=$(readlink -f "$2") +[[ -n "$IMAGE_B_PATH" ]] || die "unable to determine image B path" +[[ -f "$IMAGE_B_PATH" ]] || die "image path is not a file: '$IMAGE_B_PATH'" + +UNPACK_DIR=$(mktemp -d -p . -t diff.XXXXXXX) +[[ -d "$UNPACK_DIR" ]] || die "failed to create unpack directory '$UNPACK_DIR'" +pushd "$UNPACK_DIR" &>/dev/null || die "'pushd $UNPACK_DIR' failed" + +rm -rf ~/.aptly +import_image_into_aptly "image-a" "$IMAGE_A_PATH" +import_image_into_aptly "image-b" "$IMAGE_B_PATH" + +popd &>/dev/null || die "'popd' failed" + +# +# The repository we wish to build is the "set difference" of image A and +# image B; i.e. A - B. To do this, we perform the following steps below: +# +# 1. Create a new empty repository +# 2. Add all packages from image A to this new repository +# 3. Remove any packages that're found in image B +# +# The result of this, is a new repository containing all packages from +# image A, that are not in image B; this new repository is stored in +# ~/.aptly/public, and this can then be used by other parts of the build +# system (e.g. "upgrade-image-from-aptly-repo.sh"). +# + +aptly repo create -distribution=bionic -component=delphix upgrade-repository || + die "failed to create repository: 'upgrade-repository'" +aptly repo search image-a | xargs aptly repo copy image-a upgrade-repository || + die "failed to copy packages to repository: 'upgrade-repository'" + +# +# Here we're performing step 3 from the comment above, but since the +# "delphix-upgrade-verification" package is a bit different than most +# other packages on a Delphix appliance, we need to handle that package +# uniquely here. Specifically, we want to ensure this package is always +# contained in the resultant aptly repository (even if the package is +# the same within both image A and image B). +# +aptly repo search image-b | + grep -v "^delphix-upgrade-verification" | + xargs aptly repo remove upgrade-repository || + die "failed to remove packages from repository: 'upgrade-repository'" + +aptly publish repo -skip-contents -skip-signing upgrade-repository || + die "failed to publish repository: 'upgrade-repository'" + +[[ -d ~/.aptly/public ]] || die "failed to generate aptly repository" diff --git a/scripts/build-upgrade-image.sh b/scripts/build-upgrade-image.sh index e4bc5a94..a9520d7a 100755 --- a/scripts/build-upgrade-image.sh +++ b/scripts/build-upgrade-image.sh @@ -1,6 +1,6 @@ -#!/bin/bash -ex +#!/bin/bash -x # -# Copyright 2018 Delphix +# Copyright 2018-2020 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,167 +17,18 @@ . "${BASH_SOURCE%/*}/common.sh" -# -# This script is intended to build an upgrade image that contains all of -# the packages needed to upgrade a particular variant of the appliance, -# whichever platform it is running on. The upgrade image is a tar -# archive whose primary component is an Aptly/APT repository containing -# a version of the delphix-entire package for each supported platform -# and all of its dependencies. This repository is created by taking and -# combining all the deb.tar.gz tarballs produced by live build for this -# variant (of which there will be one per supported platform). -# - -if [[ -z "$TOP" ]]; then - echo "Must be run inside the git repsitory." - exit 1 -fi - -if [[ $# -ne 1 ]]; then - echo "Must specify a single variant." 1>&2 - exit 1 -fi - -cd "$TOP/upgrade" -APPLIANCE_VARIANT=$1 - -rm -rf ~/.aptly -rm -rf debs -mkdir debs - -# -# For upgrade images that we ship to customers, we will need to include -# the packages for every platform that we support. Building for every -# platform can be time-comsuming though, so for developer convenience, -# here we just take the artifacts from the live-build stage for whatever -# appliance versions were built (making sure that we built for at least -# one platform), and build an upgrade image from that. -# -LIVE_BUILD_OUTPUT_DIR="$TOP/live-build/build/artifacts" -if ! compgen -G "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT*.debs.tar.gz"; then - echo "No live-build artifacts found for this variant" >&2 - exit 1 -fi -for deb_tarball in "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT"*.debs.tar.gz; do - tar xf "$deb_tarball" -C debs -done - -# -# Download the delphix upgrade verification debian package. -# Note, we always pull from the "master" build of the verification -# package, no matter what the UPSTREAM_BRANCH of the appliance is that -# we're building; this is intentional. -# - -AWS_S3_URI_UPGRADE_VERIFICATION=$(resolve_s3_uri \ - "$AWS_S3_URI_UPGRADE_VERIFICATION" \ - "$AWS_S3_PREFIX_UPGRADE_VERIFICATION" \ - "devops-gate/master/upgrade-verify/master/post-push/release/latest") - -download_delphix_s3_debs "$TOP/upgrade/debs" "$AWS_S3_URI_UPGRADE_VERIFICATION" - -# -# Generate an Aptly/APT repository -# -aptly repo create -distribution=bionic -component=delphix upgrade-repository -aptly repo add upgrade-repository debs -aptly publish repo -skip-contents -skip-signing upgrade-repository - -# -# Generate the "payload" of the upgrade image, which will contain the -# upgrade scripts, as well as the Aptly/APT repository. We generate this -# "payload" tarball as a way to limit the number of files that we need -# to checksum and store in "upgrade image" tarball itself. Rather than -# the "upgrade image" tarball containing all of the files for the upgrade -# scripts and APT repository, it'll only contain the "payload.tar.gz" -# file, and a few other miscellaneous metadata files. -# -# We need to limit the files contained in the 'upgrade image" tarball -# because the signing service that we use to sign the SHA256SUMS file -# below, has a relatively small limit on the size of the file that we -# can sign with it. Thus, we need the contents of the SHA256SUMS file to -# be realtively small (and that file's size grows linearly with the -# number of files contained in the "upgrade image" tarball). -# -tar -I pigz -cf "payload.tar.gz" -C upgrade-scripts . -C ~/.aptly/public . - -cp version.info.template version.info - -# Include version information about this image. -VERSION=$(dpkg -f "$(find debs/ -name 'delphix-entire-*' | head -n 1)" version) -sed -i "s/@@VERSION@@/$VERSION/" version.info - -# -# On 6.0 versions, the appliance stack expects to find the file -# verification-version.info pre-upgrade. This necessitates the -# packaging of this file in the upgrade image along with version.info. -# -cp verification-version.info.template verification-version.info - -# Include version information about this image. -VERIFICATION_VERSION=$(dpkg -f "$(find debs/ -name 'delphix-upgrade-verification*.deb' | head -n 1)" version) -sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" verification-version.info -sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" version.info - -# -# If we're running the build manually, and these variables are not -# already set in the environment, we use an arbitrarily low falue to -# allow upgrading to the produce upgrade image from any version. -# - -if [[ -n "$DELPHIX_UPGRADE_MINIMUM_VERSION" ]]; then - sed -i \ - "s/@@MINIMUM_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_VERSION/" \ - version.info -else - sed -i "s/@@MINIMUM_VERSION@@/0.0.0.0/" version.info -fi - -if [[ -n "$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION" ]]; then - sed -i \ - "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION/" \ - version.info -else - sed -i "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/0.0.0.0/" version.info -fi - -sha256sum payload.tar.gz version.info verification-version.info prepare >SHA256SUMS - -# -# As a precaution, we disable "xtrace" so that we avoid exposing the -# DELPHIX_SIGNATURE_TOKEN environment variable contents to stdout. -# -set +o xtrace -if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then - echo "{\"data\": \"$(base64 -w 0 SHA256SUMS)\"}" >sign-request.payload - - # - # Here, we need to generate signature files for all of the appliance - # versions that'll be allowed to upgrade from, using this upgrade - # image. We rely on the user of this script to pass in this list - # of versions; generally this will be some Jenkins automation. - # - for signature_version in $DELPHIX_SIGNATURE_VERSIONS; do - curl -s -S -f -H "Content-Type: application/json" \ - -u "$DELPHIX_SIGNATURE_TOKEN" -d @sign-request.payload \ - "$DELPHIX_SIGNATURE_URL/upgrade/keyVersion/${signature_version}/sign" \ - >sign-request.response - - jq -r .signature signature.base64 - base64 -d signature.base64 >"SHA256SUMS.sig.${signature_version}" +function usage() { + echo "$(basename "$0"): $*" >&2 + echo "Usage: $(basename "$0") " + exit 2 +} - rm sign-request.response signature.base64 - done -fi -set -o xtrace +[[ -n "$TOP" ]] || die "must be run inside of the git repository" +[[ $# -gt 1 ]] && usage "too many arguments specified" +[[ $# -lt 1 ]] && usage "too few arguments specified" -# shellcheck disable=SC2046 -tar -cf "$APPLIANCE_VARIANT.upgrade.tar" \ - $(ls SHA256SUMS.sig.* 2>/dev/null) \ - SHA256SUMS \ - version.info \ - verification-version.info \ - prepare \ - payload.tar.gz +"$TOP/scripts/aptly-repo-from-debs.sh" "$1" || + die "failed to generate Aptly repository from .deb files" -mv "$APPLIANCE_VARIANT.upgrade.tar" "$TOP/build/artifacts" +"$TOP/scripts/upgrade-image-from-aptly-repo.sh" "$1" || + die "failed to generate upgrade image from Aptly repository" diff --git a/scripts/common.sh b/scripts/common.sh index ef7dd7f5..63a483e1 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -17,6 +17,11 @@ TOP=$(git rev-parse --show-toplevel 2>/dev/null) +function die() { + echo "$(basename "$0"): $*" >&2 + exit 2 +} + function resolve_s3_uri() { local pkg_uri="$1" local pkg_prefix="$2" diff --git a/scripts/upgrade-image-from-aptly-repo.sh b/scripts/upgrade-image-from-aptly-repo.sh new file mode 100755 index 00000000..9e66731e --- /dev/null +++ b/scripts/upgrade-image-from-aptly-repo.sh @@ -0,0 +1,183 @@ +#!/bin/bash -x +# +# Copyright 2018-2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is responsible for generating a Delphix appliance upgrade +# image, by consuming a previously generated Aptly/APT repository. This +# APT repository is the primary component of any upgrade image, and it +# must contain a version of the delphix-entire package for each platform +# supported, as well as all package dependencies. +# +# The Aptly repository will generally be created prior to calling this +# script, by using the "aptly-repo-from-debs.sh" script. +# + +. "${BASH_SOURCE%/*}/common.sh" + +function usage() { + echo "$(basename "$0"): $*" >&2 + echo "Usage: $(basename "$0") " + exit 2 +} + +function cleanup() { + [[ -n "$DIR" ]] && [[ -d "$DIR" ]] && rm -rf "$DIR" +} + +trap cleanup EXIT + +[[ -n "$TOP" ]] || die "must be run inside of the git repository" +[[ $# -gt 1 ]] && usage "too many arguments specified" +[[ $# -lt 1 ]] && usage "too few arguments specified" + +# +# One of the "inputs" to this script is an Aptly repository stored in +# the "~/.aptly/public" directory. Thus, we do minimal verification of +# this input, and see if the directory exists before proceeding. The +# intention is for this directory to have been previously generated by +# another part of the build system; e.g. via "aptly-repo-from-debs.sh". +# +[[ -d ~/.aptly/public ]] || die "aptly repository not found" + +DIR=$(mktemp -d -t upgrade-image-from-aptly-repo.XXXXXXX) +[[ -d "$DIR" ]] || die "unable to create temporary directory" +pushd "$DIR" &>/dev/null || die "'pushd $DIR' failed" + +# +# Generate the "payload" of the upgrade image, which will contain the +# upgrade scripts, as well as the Aptly/APT repository. We generate this +# "payload" tarball as a way to limit the number of files that we need +# to checksum and store in "upgrade image" tarball itself. Rather than +# the "upgrade image" tarball containing all of the files for the upgrade +# scripts and APT repository, it'll only contain the "payload.tar.gz" +# file, and a few other miscellaneous metadata files. +# +# We need to limit the files contained in the 'upgrade image" tarball +# because the signing service that we use to sign the SHA256SUMS file +# below, has a relatively small limit on the size of the file that we +# can sign with it. Thus, we need the contents of the SHA256SUMS file to +# be relatively small (and that file's size grows linearly with the +# number of files contained in the "upgrade image" tarball). +# +tar -I pigz -cf "payload.tar.gz" \ + -C "$TOP/upgrade/upgrade-scripts" . \ + -C ~/.aptly/public . || + die "failed to generate payload.tar.gz" + +cp "$TOP/upgrade/version.info.template" version.info || + die "failed to copy version.info.template" + +VERSION=$(dpkg -f "$(find ~/.aptly/public -type f -name 'delphix-entire-*' | head -n 1)" version) +[[ -n "$VERSION" ]] || die "failed to determine delphix-entire version" +sed -i "s/@@VERSION@@/$VERSION/" version.info || + die "failed to set VERSION in version.info file" + +# +# On 6.0 versions, the virtualization application expects to find the +# "verification-version.info" file within the unpacked upgrade image +# directory. Thus, to satisfy those requirements, we must provide that +# file, in addition to the usual "version.info". +# +cp "$TOP/upgrade/verification-version.info.template" verification-version.info || + die "failed to copy verifiction-version.info.template" + +VERIFICATION_VERSION=$(dpkg -f "$(find ~/.aptly/public -type f -name 'delphix-upgrade-verification*.deb' | head -n 1)" version) +[[ -n "$VERIFICATION_VERSION" ]] || die "failed to determine verification version" +sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" verification-version.info || + die "failed to set VERIFICATION_VERSION in verification-version.info file" +sed -i "s/@@VERIFICATION_VERSION@@/$VERIFICATION_VERSION/" version.info || + die "failed to set VERIFICATION_VERSION in version.info file" + +# +# If we're running the build manually, and these variables are not +# already set in the environment, we use an arbitrarily low value to +# allow upgrading to the produced upgrade image from any version. +# + +if [[ -n "$DELPHIX_UPGRADE_MINIMUM_VERSION" ]]; then + sed -i \ + "s/@@MINIMUM_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_VERSION/" \ + version.info || + die "failed to set MINIMUM_VERSION in version.info" +else + sed -i "s/@@MINIMUM_VERSION@@/0.0.0.0/" version.info || + die "failed to set MINIMUM_VERSION in version.info" +fi + +if [[ -n "$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION" ]]; then + sed -i \ + "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION/" \ + version.info || + die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" +else + sed -i "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/0.0.0.0/" version.info || + die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" +fi + +cp "$TOP/upgrade/prepare" prepare || die "failed to copy prepare" + +sha256sum payload.tar.gz version.info verification-version.info prepare >SHA256SUMS || + die "failed to generate SHA256SUMS file" + +# +# As a precaution, we disable "xtrace" so that we avoid exposing the +# DELPHIX_SIGNATURE_TOKEN environment variable contents to stdout. +# +set +o xtrace +if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then + echo "{\"data\": \"$(base64 -w 0 SHA256SUMS)\"}" >sign-request.payload || + die "failed to generate sigh-request.payload file" + + # + # Here, we need to generate signature files for all of the appliance + # versions that'll be allowed to upgrade from, using this upgrade + # image. We rely on the user of this script to pass in this list + # of versions; generally this will be some Jenkins automation. + # + for signature_version in $DELPHIX_SIGNATURE_VERSIONS; do + curl -s -S -f -H "Content-Type: application/json" \ + -u "$DELPHIX_SIGNATURE_TOKEN" -d @sign-request.payload \ + "$DELPHIX_SIGNATURE_URL/upgrade/keyVersion/${signature_version}/sign" \ + >sign-request.response || + die "failed to generate sign-request.response file" + + jq -r .signature signature.base64 || + die "failed to generate signature.base64 file" + base64 -d signature.base64 >"SHA256SUMS.sig.${signature_version}" || + die "failed to decode signature.base64 file" + + rm sign-request.response signature.base64 || + die "failed to remove temporary signature files" + done +fi +set -o xtrace + +# shellcheck disable=SC2046 +tar -cf "$1.upgrade.tar" \ + $(ls SHA256SUMS.sig.* 2>/dev/null) \ + SHA256SUMS \ + version.info \ + verification-version.info \ + prepare \ + payload.tar.gz || + die "failed to generate upgrade image" + +mkdir -p "$TOP/build/artifacts" || + die "failed to create build artifacts directory" + +mv "$1.upgrade.tar" "$TOP/build/artifacts" || + die "failed to move upgrade image to build artifacts directory" From 1a5f3a2fb490002e0f840f27a418c1408934b6f0 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 17 Dec 2020 22:17:12 -0500 Subject: [PATCH 059/197] DLPX-73378 [Backport of DLPX-73299 to 6.0.6.0] appliance-build: Fetch upgrade-verify package from combined-packages bundle (#515) --- build.gradle | 3 +- live-build/build.gradle | 2 +- scripts/aptly-repo-from-debs.sh | 22 ++--- scripts/build-ancillary-repository.sh | 72 +++++++---------- scripts/common.sh | 111 ++++++++++++++++++-------- 5 files changed, 119 insertions(+), 91 deletions(-) diff --git a/build.gradle b/build.gradle index 74fcef90..ba3dffef 100644 --- a/build.gradle +++ b/build.gradle @@ -53,8 +53,7 @@ for (variant in allVariants) { for (envVar in ["DELPHIX_PLATFORMS", "AWS_S3_URI_LIVEBUILD_ARTIFACTS", - "AWS_S3_URI_UPGRADE_VERIFICATION", - "AWS_S3_PREFIX_UPGRADE_VERIFICATION"]) { + "AWS_S3_URI_COMBINED_PACKAGES"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) } diff --git a/live-build/build.gradle b/live-build/build.gradle index c00e808f..a7ae5e85 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -21,7 +21,7 @@ apply from: "${rootProject.projectDir}/gradle-lib/util.gradle" task ancillaryRepository(type: Exec) { inputs.file "${rootProject.projectDir}/scripts/build-ancillary-repository.sh" - for (envVar in ["COMBINED_PACKAGES_S3_URL"]) { + for (envVar in ["AWS_S3_URI_COMBINED_PACKAGES"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) } diff --git a/scripts/aptly-repo-from-debs.sh b/scripts/aptly-repo-from-debs.sh index 5874417b..f7628988 100755 --- a/scripts/aptly-repo-from-debs.sh +++ b/scripts/aptly-repo-from-debs.sh @@ -61,18 +61,22 @@ for deb_tarball in "$LIVE_BUILD_OUTPUT_DIR/$APPLIANCE_VARIANT"*.debs.tar.gz; do done # -# Download the delphix upgrade verification debian package. -# Note, we always pull from the "master" build of the verification -# package, no matter what the UPSTREAM_BRANCH of the appliance is that -# we're building; this is intentional. +# Download the delphix upgrade verification debian package, stored in the +# combined-packages bundle. # +AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ + "$AWS_S3_URI_COMBINED_PACKAGES" \ + "devops-gate/master/linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") -AWS_S3_URI_UPGRADE_VERIFICATION=$(resolve_s3_uri \ - "$AWS_S3_URI_UPGRADE_VERIFICATION" \ - "$AWS_S3_PREFIX_UPGRADE_VERIFICATION" \ - "devops-gate/master/upgrade-verify/master/post-push/release/latest") +WORK_DIRECTORY=$(mktemp -d -p "$TOP/upgrade" tmp.pkgs.XXXXXXXXXX) -download_delphix_s3_debs "$TOP/upgrade/debs" "$AWS_S3_URI_UPGRADE_VERIFICATION" +download_combined_packages_artifacts "$AWS_S3_URI_COMBINED_PACKAGES" \ + "$WORK_DIRECTORY" upgrade-verify + +extract_debs_into_dir "$WORK_DIRECTORY/packages/upgrade-verify" \ + "$TOP/upgrade/debs" + +rm -rf "$WORK_DIRECTORY" # # Generate an Aptly/APT repository diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 8ceeb7ce..ae512f26 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -61,37 +61,6 @@ function build_ancillary_repository() { EOF } -# -# The packages produced by Delphix are stored in Amazon S3. -# Thus, in order to populate the ancillary repository with these -# packages, they must be downloaded from S3, so they can be then -# inserted into the Aptly repository. -# -# Here, we determine the URI of each of the Delphix packages, and -# then use these URIs to download the packages later. Making this -# determination is a little complex, and is dependent on the policy set -# forth by the teams producing and storing the packages. -# -# With that said, there's three main methods of influencing the URI from -# which the packages are downloaded: -# -# 1. If the package specific AWS_S3_URI environment variable is provided -# (e.g. AWS_S3_URI_UPGRADE_VERIFICATION), then this URI will be used to -# download the package. This is the simplest case, and enables the -# user of this script to directly influence this script. -# -# 2. If the package specific AWS_S3_PREFIX environment variable is -# provided (e.g. AWS_S3_PREFIX_UPGRADE_VERIFICATION), then this value is -# used to build the URI that will be used based on the default S3 -# bucket that is used. -# -# 3. If nether the package specific AWS_S3_URI nor AWS_S3_PREFIX -# variables are provided, then logic kicks in to attempt to -# dynamically determine the URI of the most recently built package, -# and then uses that URI. This way, a naive user can not set any -# environment variables, and the script will work as expected. -# - # # Set UPSTREAM_BRANCH. This will determine which version of the linux package # mirror is used. @@ -115,32 +84,45 @@ else fi echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" +# +# The packages produced by Delphix are stored in Amazon S3. +# Thus, in order to populate the ancillary repository with these +# packages, they must be downloaded from S3, so they can be then +# inserted into the Aptly repository. +# +# All the Delphix-built packages consumed by appliance-build are compiled by +# the combine-packages job. If a combine-packages URI is provided, fetch the +# packages from there, otherwise determine the latest combined-packages URI +# automatically. +# + AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ - "$AWS_S3_URI_COMBINED_PACKAGES" "" \ + "$AWS_S3_URI_COMBINED_PACKAGES" \ "devops-gate/master/linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") +mkdir -p "$TOP/build" +WORK_DIRECTORY=$(mktemp -d -p "$TOP/build" tmp.pkgs.XXXXXXXXXX) + # -# All package files will be placed into this temporary directory, such -# that we can later point Aptly at this directory to build the Aptly/APT -# repository. +# Download all package artifacts built by Delphix, which includes debs and +# metadata. # -mkdir -p "$TOP/build" -PKG_DIRECTORY=$(mktemp -d -p "$TOP/build" tmp.pkgs.XXXXXXXXXX) +mkdir -p "$WORK_DIRECTORY/artifacts" +download_combined_packages_artifacts "$AWS_S3_URI_COMBINED_PACKAGES" \ + "$WORK_DIRECTORY/artifacts" # -# Now that we've determined the URI of the Delphix-built packages, we can -# download them. +# Find all debs and put them into a directory that will be fed into Aptly. # -download_delphix_s3_debs_multidir "$PKG_DIRECTORY" "$AWS_S3_URI_COMBINED_PACKAGES/packages" +mkdir -p "$WORK_DIRECTORY/debs" +extract_debs_into_dir "$WORK_DIRECTORY/artifacts" "$WORK_DIRECTORY/debs" # -# Now that our temporary package directory has been populated with all -# first-party packages needed by live-build, we use this directory to -# build up our Aptly/APT ancillary repository. After this function +# Build up our Aptly/APT ancillary repository. After this function # completes, there should be a directory named "ancillary-repository" at # the top level of the git repository, that can later be "aptly # serve"-ed and consumed by live-build. # -build_ancillary_repository "$PKG_DIRECTORY" +build_ancillary_repository "$WORK_DIRECTORY/debs" -rm -rf "$PKG_DIRECTORY" +rm -rf "$WORK_DIRECTORY" diff --git a/scripts/common.sh b/scripts/common.sh index 63a483e1..99f0942c 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -15,6 +15,7 @@ # limitations under the License. # +# shellcheck disable=SC2034 TOP=$(git rev-parse --show-toplevel 2>/dev/null) function die() { @@ -24,8 +25,7 @@ function die() { function resolve_s3_uri() { local pkg_uri="$1" - local pkg_prefix="$2" - local latest_subprefix="$3" + local latest_subprefix="$2" local bucket="snapshot-de-images" local jenkinsid="jenkins-ops" @@ -33,10 +33,6 @@ function resolve_s3_uri() { if [[ -n "$pkg_uri" ]]; then resolved_uri="$pkg_uri" - elif [[ "$pkg_prefix" == s3* ]]; then - resolved_uri="$pkg_prefix" - elif [[ -n "$pkg_prefix" ]]; then - resolved_uri="s3://$bucket/$pkg_prefix" elif [[ -n "$latest_subprefix" ]]; then aws s3 cp --quiet \ "s3://$bucket/builds/$jenkinsid/$latest_subprefix" . @@ -55,40 +51,87 @@ function resolve_s3_uri() { fi } -function download_delphix_s3_debs() { - local pkg_directory="$1" - local S3_URI="$2" - local tmp_directory - - tmp_directory=$(mktemp -d -p "$TOP/build" tmp.s3-debs.XXXXXXXXXX) - pushd "$tmp_directory" &>/dev/null - - aws s3 sync --only-show-errors "$S3_URI" . - sha256sum -c --strict SHA256SUMS - - mv ./*deb "$pkg_directory/" - - popd &>/dev/null - rm -rf "$tmp_directory" -} +# +# Given an S3 URI pointing to combined-packages artifacts, download all of its +# artifacts to target directory. If a package name is passed as an argument, +# then only copy the artifacts for that package. +# +# When the combine-packages Jenkins job generates artifacts, it does not +# copy around the artifacts for individual packages. Rather, it creates a +# COMPONENTS file that has links to each package's artifacts. +# +# When appliance-build is ran via Jenkins, the Jenkins job copies the original +# combined-packages artifacts to a new S3 location, then dereferences the +# COMPONENTS file and copies all individual package artifacts into a +# "packages" directory created under that new S3 location. Jenkins then passes +# that combined-packages URI to live-build. +# +# Thus if a "packages" directory is found under the combined-packages S3 URI, +# we assume that the dereferencing has already been done and so we just sync +# the whole directory. Otherwise, we must dereference the COMPONENTS file here +# and fetch the artifacts for each package. +# +# Here are the files that are expected to be found after the download. +# / +# COMPONENTS +# ... (some other metadata files) +# packages/ +# package1/ +# ... (package 1 artifacts) +# package2/ +# ... (package 2 artifacts) +# ... (remaining packages' artifacts) +# +# shellcheck disable=SC2164 +function download_combined_packages_artifacts() { + local combined_pkgs_uri="$1" + local target_dir="$2" + local pkg="$3" -function download_delphix_s3_debs_multidir() { - local pkg_directory="$1" - local S3_URI="$2" - local tmp_directory + pushd "$target_dir" &>/dev/null - tmp_directory=$(mktemp -d -p "$TOP/build" tmp.s3-debs.XXXXXXXXXX) - pushd "$tmp_directory" &>/dev/null + if [[ -n "$pkg" ]]; then + aws s3 sync --exclude 'packages/*' --include "packages/$pkg/*" \ + --only-show-errors "$combined_pkgs_uri" . + else + aws s3 sync --only-show-errors "$combined_pkgs_uri" . + fi - aws s3 sync --only-show-errors "$S3_URI" . + if [[ -d packages ]]; then + popd &>/dev/null + return + fi - for subdir in */; do - pushd "$subdir" &>/dev/null + [[ -f COMPONENTS ]] || die "COMPONENTS file missing." + mkdir packages + pushd packages &>/dev/null + + local pkgname s3uri + while read -r line; do + pkgname=$(echo "$line" | cut -d: -f 1 | tr -d '[:space:]') + s3uri=$(echo "$line" | cut -d: -f 2- | tr -d '[:space:]') + [[ -n "$pkg" ]] && [[ "$pkg" != "$pkgname" ]] && continue + mkdir "$pkgname" + pushd "$pkgname" &>/dev/null + aws s3 sync --only-show-errors "$s3uri" . sha256sum -c --strict SHA256SUMS - mv ./*deb "$pkg_directory/" popd &>/dev/null - done + done <../COMPONENTS popd &>/dev/null - rm -rf "$tmp_directory" + popd &>/dev/null +} + +# +# Find all .deb and .ddeb packages in source directory tree and move them +# to target directory. +# +function extract_debs_into_dir() { + local source_dir="$1" + local target_dir="$2" + + [[ -d "$target_dir" ]] || + die "'$target_dir' must be an existing directory" + find "$source_dir" -name '*.deb' -exec mv {} "$target_dir" \; + find "$source_dir" -name '*.ddeb' -exec mv {} "$target_dir" \; } From cd326e266415d50575426d46c682369831ff2a78 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 29 Jan 2021 13:26:33 -0800 Subject: [PATCH 060/197] TOOL-10893 High disk space usage for audit logs threatens the success of future upgrades (#518) (#519) --- .../hooks/vm-artifacts/90-raw-disk-image.binary | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index a68e0d57..2bcc1f24 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -43,13 +43,18 @@ die() { # We want to use different sized rpool depending on if we're building a # disk image meant for internal use, or external (i.e. customer) use. # +# The only exception to this is our "dcenter" variant. While that +# variant is only used internally, we use it in a way that more +# resembles our external variants, so we want the rpool size for +# the dcenter images to match our external images. +# case "$APPLIANCE_VARIANT" in +external-* | internal-dcenter) + RAW_DISK_SIZE_GB=127 + ;; internal-*) RAW_DISK_SIZE_GB=70 ;; -external-*) - RAW_DISK_SIZE_GB=127 - ;; *) die "Invalid variant specified: '$APPLIANCE_VARIANT'" ;; From c377bbcb938f75543a950fbca4aaac3562d93b04 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Mon, 8 Feb 2021 13:37:33 -0600 Subject: [PATCH 061/197] CP-4470 [Backport of CP-4449 to 6.0.7.0] use pvscsi by default (#522) CP-4471 [Backport of CP-4451 to 6.0.7.0] Bump OVA hardware version to HWv11 --- live-build/config/hooks/vm-artifacts/template.ovf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/template.ovf b/live-build/config/hooks/vm-artifacts/template.ovf index 78edfb51..68ff5ddc 100644 --- a/live-build/config/hooks/vm-artifacts/template.ovf +++ b/live-build/config/hooks/vm-artifacts/template.ovf @@ -42,7 +42,7 @@ Virtual Hardware Family 0 @@VM_NAME@@ - vmx-10 + vmx-11 hertz * 10^6 @@ -65,7 +65,7 @@ SCSI Controller SCSI Controller 0 3 - lsilogic + VirtualSCSI 6 @@ -151,7 +151,7 @@ A human-readable annotation - Delphix Appliance, VM Hardware Version 10 + Delphix Appliance, VM Hardware Version 11 From 3a489278014883e61aff9ee6ee197c7e192de9e8 Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Thu, 11 Feb 2021 05:45:51 -0800 Subject: [PATCH 062/197] Backporting #523, #524, #525, and #526. (#527) --- .../appliance-build.dcenter/tasks/main.yml | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 24009464..987e286d 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -28,23 +28,17 @@ - nfs-common-dbgsym - nfs-kernel-server - nfs-kernel-server-dbgsym - - python-dbg - - python-dev - - python-ldap - - python-paramiko - - python-pip - - python-pyvmomi - - python-six - - python-tenacity - - python2.7 - python3 - python3-dbg - python3-dev - python3-ldap + - python3-marshmallow + - python3-marshmallow-doc - python3-pip - python3-pyvmomi - python3-six - python3-tenacity + - python3-toml - python3-venv - targetcli-fb - telnet @@ -54,12 +48,6 @@ retries: 3 delay: 60 -- pip: - name: pysphere - extra_args: --index-url http://artifactory.delphix.com/artifactory/api/pypi/delphix-virtual-pypi/simple/ --trusted-host artifactory.delphix.com --no-cache-dir - version: 0.1.8 - executable: pip2 - - git: repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' version: master @@ -89,15 +77,6 @@ /tmp/dcenter_dhcp_config/named.zone r, /tmp/dcenter_dhcp_config/named.pid w, -# -# Dcenter systems use static addresses so modify cloud.cfg to preserve -# their hostname. -# -- lineinfile: - path: /etc/cloud/cloud.cfg - regexp: '^preserve_hostname: false' - line: 'preserve_hostname: true' - # # The default setting for the number of nfs threads is too low. To # improve performance we reset the value to 64 which mimics what @@ -111,4 +90,4 @@ - { regexp: '^RPCNFSDCOUNT=', line: 'RPCNFSDCOUNT=64' } - { regexp: '^RPCMOUNTDOPTS=', line: 'RPCMOUNTDOPTS="--num-threads=5 --manage-gids"' } -- command: systemctl disable bind9.service +- command: systemctl disable bind9.service isc-dhcp-server.service isc-dhcp-server6.service From d15212f688dfd4f2d6f51624f510e6c56669d0d2 Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Thu, 11 Feb 2021 13:30:09 -0800 Subject: [PATCH 063/197] Backporting #523, #524, #525, and #526. (#528) --- .../appliance-build.dcenter/tasks/main.yml | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 24009464..987e286d 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -28,23 +28,17 @@ - nfs-common-dbgsym - nfs-kernel-server - nfs-kernel-server-dbgsym - - python-dbg - - python-dev - - python-ldap - - python-paramiko - - python-pip - - python-pyvmomi - - python-six - - python-tenacity - - python2.7 - python3 - python3-dbg - python3-dev - python3-ldap + - python3-marshmallow + - python3-marshmallow-doc - python3-pip - python3-pyvmomi - python3-six - python3-tenacity + - python3-toml - python3-venv - targetcli-fb - telnet @@ -54,12 +48,6 @@ retries: 3 delay: 60 -- pip: - name: pysphere - extra_args: --index-url http://artifactory.delphix.com/artifactory/api/pypi/delphix-virtual-pypi/simple/ --trusted-host artifactory.delphix.com --no-cache-dir - version: 0.1.8 - executable: pip2 - - git: repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' version: master @@ -89,15 +77,6 @@ /tmp/dcenter_dhcp_config/named.zone r, /tmp/dcenter_dhcp_config/named.pid w, -# -# Dcenter systems use static addresses so modify cloud.cfg to preserve -# their hostname. -# -- lineinfile: - path: /etc/cloud/cloud.cfg - regexp: '^preserve_hostname: false' - line: 'preserve_hostname: true' - # # The default setting for the number of nfs threads is too low. To # improve performance we reset the value to 64 which mimics what @@ -111,4 +90,4 @@ - { regexp: '^RPCNFSDCOUNT=', line: 'RPCNFSDCOUNT=64' } - { regexp: '^RPCMOUNTDOPTS=', line: 'RPCMOUNTDOPTS="--num-threads=5 --manage-gids"' } -- command: systemctl disable bind9.service +- command: systemctl disable bind9.service isc-dhcp-server.service isc-dhcp-server6.service From 2d7e99486e36ab3ba8da14af5ead78d95c6b24b5 Mon Sep 17 00:00:00 2001 From: Raul Santelices Date: Tue, 16 Feb 2021 11:06:53 -0500 Subject: [PATCH 064/197] Fix branch.config --- branch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/branch.config b/branch.config index 9e572e67..8b13b1e8 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/stage" +UPSTREAM_BRANCH="6.0/release" From e6d1f7fadafecbe8df832e1fb306178287cf8fdb Mon Sep 17 00:00:00 2001 From: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Date: Fri, 5 Mar 2021 10:09:31 -0800 Subject: [PATCH 065/197] DLPX-71882 [Backport of DLPX-71833 to 6.0/stage] ui-precommit fails to start ChromeHeadless (#530) --- .../appliance-build.unittest-internal/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index 1f0430ee..f5c9d47e 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -25,6 +25,7 @@ - git - python-minimal - chromium-browser + - libxss1 state: present - user: From 51c5e12f19d201eeb1c16cbb3f6c2c70d860b234 Mon Sep 17 00:00:00 2001 From: Matt Skinner Date: Fri, 12 Mar 2021 15:35:41 -0500 Subject: [PATCH 066/197] Initial changes to support new hotfix workflow(s) (#507) (#531) * Support "hotfix-version" property on build and upgrade * Support hotfix version in upgrade image * Set hotfix version via "execute" script Co-authored-by: Prakash Surya --- build.gradle | 3 +- live-build/build.gradle | 1 + .../vm-artifacts/90-raw-disk-image.binary | 6 +++ scripts/upgrade-image-from-aptly-repo.sh | 9 ++++ upgrade/upgrade-scripts/common.sh | 47 +++++++++++++++++-- upgrade/upgrade-scripts/execute | 18 +++++-- upgrade/upgrade-scripts/upgrade-container | 10 ++-- upgrade/version.info.template | 5 ++ 8 files changed, 88 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index ba3dffef..067c29dd 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,8 @@ for (variant in allVariants) { } } - for (envVar in ["DELPHIX_PLATFORMS", + for (envVar in ["DELPHIX_PLATFORMS", + "DELPHIX_HOTFIX_VERSION", "AWS_S3_URI_LIVEBUILD_ARTIFACTS", "AWS_S3_URI_COMBINED_PACKAGES"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) diff --git a/live-build/build.gradle b/live-build/build.gradle index a7ae5e85..89717ece 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -101,6 +101,7 @@ for (variant in allVariants) { for (envVar in ["APPLIANCE_PASSWORD", "DELPHIX_APPLIANCE_VERSION", + "DELPHIX_HOTFIX_VERSION", "DELPHIX_PACKAGE_MIRROR_MAIN", "DELPHIX_PACKAGE_MIRROR_SECONDARY", "DELPHIX_SIGNATURE_URL", diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 2bcc1f24..2609306a 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -136,6 +136,12 @@ zfs create \ -o "com.delphix:current-version=$DELPHIX_APPLIANCE_VERSION" \ "rpool/ROOT/$FSNAME" +if [[ -n "$DELPHIX_HOTFIX_VERSION" ]]; then + zfs set \ + "com.delphix:hotfix-version=$DELPHIX_HOTFIX_VERSION" \ + "rpool/ROOT/$FSNAME" +fi + zfs create \ -o canmount=noauto \ -o mountpoint=/ \ diff --git a/scripts/upgrade-image-from-aptly-repo.sh b/scripts/upgrade-image-from-aptly-repo.sh index 9e66731e..7eb18aa1 100755 --- a/scripts/upgrade-image-from-aptly-repo.sh +++ b/scripts/upgrade-image-from-aptly-repo.sh @@ -86,6 +86,15 @@ VERSION=$(dpkg -f "$(find ~/.aptly/public -type f -name 'delphix-entire-*' | hea sed -i "s/@@VERSION@@/$VERSION/" version.info || die "failed to set VERSION in version.info file" +# +# The DELPHIX_HOTFIX_VERSION variable is optional, and thus it may not +# be set at this point. That is by design, and when that's the case, we +# still need to do this replacement, such that the version information +# file properly reflects an empty value for the hotfix version. +# +sed -i "s/@@HOTFIX@@/$DELPHIX_HOTFIX_VERSION/" version.info || + die "failed to set HOTFIX in version.info file" + # # On 6.0 versions, the virtualization application expects to find the # "verification-version.info" file within the unpacked upgrade image diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 7640b41f..b917a220 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -47,6 +47,7 @@ umask 0022 # PROP_CURRENT_VERSION="com.delphix:current-version" PROP_INITIAL_VERSION="com.delphix:initial-version" +PROP_HOTFIX_VERSION="com.delphix:hotfix-version" # # To better enable root cause analysis of any upgrade failures, we @@ -174,27 +175,45 @@ function get_snapshot_clones() { zfs get clones -Hpo value "$1" } -function get_current_version() { +function get_version_property() { + [[ -n "$1" ]] || die "version property not specified" + local DATASET DATASET="$(get_mounted_rootfs_container_dataset)" [[ -n "$DATASET" ]] || die "could not determine mounted rootfs container dataset" local VERSION - VERSION=$(zfs get -Hpo value "$PROP_CURRENT_VERSION" "$DATASET") + VERSION=$(zfs get -Hpo value "$1" "$DATASET") [[ -n "$VERSION" && "$VERSION" != "-" ]] || - die "could not determine current version for '$DATASET'" + die "could not get version property '$1' for dataset '$DATASET'" echo "$VERSION" } -function copy_dataset_property() { +function get_current_version() { + get_version_property "$PROP_CURRENT_VERSION" +} + +function get_hotfix_version() { + get_version_property "$PROP_HOTFIX_VERSION" +} + +function copy_required_dataset_property() { local PROP_NAME="$1" local SRC_DATASET="$2" local DST_DATASET="$3" local PROP_VALUE PROP_VALUE=$(zfs get -Hpo value "$PROP_NAME" "$SRC_DATASET") + + # + # Unlike the "copy_optional_dataset_property" function, if the + # property does not exist on the dataset, we return an error. + # This is useful for properties that should always exist on the + # dataset, in which case failing to retrieve the original value + # should always be treated as an exception. + # [[ -n "$PROP_VALUE" && "$PROP_VALUE" != "-" ]] || die "failed to get property '$PROP_NAME' for '$SRC_DATASET'" @@ -202,6 +221,26 @@ function copy_dataset_property() { die "failed to set property '$PROP_NAME=$PROP_VALUE' for '$DST_DATASET'" } +function copy_optional_dataset_property() { + local PROP_NAME="$1" + local SRC_DATASET="$2" + local DST_DATASET="$3" + local PROP_VALUE + + PROP_VALUE=$(zfs get -Hpo value "$PROP_NAME" "$SRC_DATASET") + + # + # Unlike the "copy_required_dataset_property" function, if the + # property does not exist on the dataset, we return without + # copying the property. This is useful if the property needs to + # be copied when it exists, and ignored otherwise. + # + [[ -n "$PROP_VALUE" && "$PROP_VALUE" != "-" ]] || return + + zfs set "$PROP_NAME=$PROP_VALUE" "$DST_DATASET" || + die "failed to set property '$PROP_NAME=$PROP_VALUE' for '$DST_DATASET'" +} + function compare_versions() { dpkg --compare-versions "$@" } diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 7461592d..2ff40480 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -98,14 +98,16 @@ if [[ -n "$CURRENT_VERSION" ]]; then # # The properties for a snapshot will inherit its values from the - # parent dataset by default. Thus, in order for this property to - # reflect the version contained in this specific snapshot, we + # parent dataset by default. Thus, in order for these properties + # to reflect the versions contained in this specific snapshot, we # need to explicity set the property here. Otherwise, when the # property changes on the dataset (i.e. when we change it later # in this script), it'd change for this snapshot, even though # the contents of the snapshot wouldn't change. # - copy_dataset_property "$PROP_CURRENT_VERSION" \ + copy_required_dataset_property "$PROP_CURRENT_VERSION" \ + "$ROOTFS_CONTAINER" "$ROOTFS_CONTAINER@execute-upgrade.$UNIQUE" + copy_optional_dataset_property "$PROP_HOTFIX_VERSION" \ "$ROOTFS_CONTAINER" "$ROOTFS_CONTAINER@execute-upgrade.$UNIQUE" fi @@ -444,6 +446,16 @@ zfs set "$PROP_CURRENT_VERSION=$VERSION" "$ROOTFS_CONTAINER" || die "failed to set property '$PROP_CURRENT_VERSION'" \ "to '$VERSION' for '$ROOTFS_CONTAINER'" +if [[ -n "$HOTFIX" ]]; then + zfs set "$PROP_HOTFIX_VERSION=$HOTFIX" "$ROOTFS_CONTAINER" || + die "failed to set property '$PROP_HOTFIX_VERSION'" \ + "to '$HOTFIX' for '$ROOTFS_CONTAINER'" +else + zfs inherit "$PROP_HOTFIX_VERSION" "$ROOTFS_CONTAINER" || + die "failed to inherit property '$PROP_HOTFIX_VERSION'" \ + "for '$ROOTFS_CONTAINER'" +fi + # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index a18ed7d4..199e7f0d 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -115,7 +115,9 @@ function create_upgrade_container() { # specific snapshot, we need to explicity set the # property here. # - copy_dataset_property "$PROP_CURRENT_VERSION" \ + copy_required_dataset_property "$PROP_CURRENT_VERSION" \ + "$ROOTFS_DATASET" "$ROOTFS_DATASET@$SNAPSHOT_NAME" + copy_optional_dataset_property "$PROP_HOTFIX_VERSION" \ "$ROOTFS_DATASET" "$ROOTFS_DATASET@$SNAPSHOT_NAME" ;; rollback) @@ -141,9 +143,11 @@ function create_upgrade_container() { # Thus, for these, we want the new container's version # properties to match the dataset being cloned. # - copy_dataset_property "$PROP_INITIAL_VERSION" \ + copy_required_dataset_property "$PROP_INITIAL_VERSION" \ + "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" + copy_required_dataset_property "$PROP_CURRENT_VERSION" \ "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" - copy_dataset_property "$PROP_CURRENT_VERSION" \ + copy_optional_dataset_property "$PROP_HOTFIX_VERSION" \ "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" zfs clone \ diff --git a/upgrade/version.info.template b/upgrade/version.info.template index 4259287d..25e38262 100644 --- a/upgrade/version.info.template +++ b/upgrade/version.info.template @@ -34,6 +34,11 @@ # VERSION=@@VERSION@@ +# +# The hotfix version contained in the upgrade image (if any) +# +HOTFIX=@@HOTFIX@@ + # # The minimum product version an engine must have installed, in order to # upgrade using this upgrade image. From 756c03490bea2af3be1760e855bdc7760c126304 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 7 Apr 2021 08:51:17 -0700 Subject: [PATCH 067/197] DLPX-75090 [Backport of DLPX-75089 to 6.0.8.0] Upgrade verification should not enable nfs services (#535) --- .github/scripts/install-ansible-lint.sh | 2 ++ upgrade/upgrade-scripts/upgrade-container | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/scripts/install-ansible-lint.sh b/.github/scripts/install-ansible-lint.sh index 3ac7e126..c36c6568 100755 --- a/.github/scripts/install-ansible-lint.sh +++ b/.github/scripts/install-ansible-lint.sh @@ -1,5 +1,7 @@ #!/bin/bash -ex +sudo apt-get install ansible + git clone https://github.com/willthames/ansible-lint /opt/ansible-lint cd /opt/ansible-lint git checkout v3.4.21 diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 199e7f0d..c38abc98 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -262,6 +262,27 @@ function create_upgrade_container() { rpool/crashdump /var/crash zfs defaults,x-systemd.before=zfs-import-cache.service,x-systemd.before=kdump-tools.service 0 0 EOF + # + # DLPX-75089 - Since older versions of Delphix did not properly + # disable the NFS services within the upgrade container, we have + # to dynamically disable them here. This way, when the container + # is started, those NFS services will not run (no matter the + # version of Delphix we're upgrading from). + # + for svc in "nfs-mountd" "nfs-server" "rpc-statd" "rpc-statd-notify"; do + [[ -e "$DIRECTORY/lib/systemd/system/$svc.service.d/override.conf" ]] && + continue + + mkdir -p "$DIRECTORY/lib/systemd/system/$svc.service.d" || + die "failed to create override directory for '$svc' service" + + cat >"$DIRECTORY/lib/systemd/system/$svc.service.d/override.conf" <<-EOF || + [Unit] + ConditionVirtualization=!container + EOF + die "failed to create override file for '$svc' service" + done + mkdir -p "/etc/systemd/nspawn" || die "failed to create directory: '/etc/systemd/nspawn'" From a96aa5b9786b75b467d88bc037410c010504343c Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Mon, 12 Apr 2021 23:31:28 -0700 Subject: [PATCH 068/197] DLPX-75232 [Backport of DLPX-75229 to 6.0.8.0] finalize() should not rely on properties in upgrade.properties (#539) --- upgrade/upgrade-scripts/upgrade | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 8069100a..8973b027 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -375,20 +375,6 @@ function finalize() { return fi - source_upgrade_properties - - [[ -n "$UPGRADE_TYPE" ]] || - die "variable UPGRADE_TYPE is not set; is upgrade in progress?" - [[ -n "$UPGRADE_BASE_VERSION" ]] || - die "variable UPGRADE_BASE_VERSION is not set" - - case "$UPGRADE_TYPE" in - DEFERRED | FULL | ROLLBACK) ;; - *) - die "finalize is not supported for upgrade type: '$UPGRADE_TYPE'" - ;; - esac - # # If we've reached this point, it means we're finalizing an # ugprade from a linux-based release, to another linux-based From 0d5b1eb4cb19219e071e52c7cbfd71e8ca744c6d Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Tue, 20 Apr 2021 13:07:17 -0700 Subject: [PATCH 069/197] DLPX-75258 [Backport of DLPX-73923 to 6.0.9.0] Persist iSCSI initiator files during not-in-place upgrade (#542) --- upgrade/upgrade-scripts/upgrade-container | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index c38abc98..b7e9610a 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -648,6 +648,7 @@ function migrate_configuration() { /etc/hostid /etc/hostname /etc/hosts + /etc/iscsi/initiatorname.iscsi /etc/machine-id /etc/netplan/10-delphix.yaml /etc/resolv.conf @@ -708,6 +709,7 @@ function migrate_configuration() { while read -r dir; do migrate_dir "$dir" done <<-EOF + /etc/iscsi/nodes /var/lib/nfs /var/target/pr EOF From dc09efc9d15e91a9f39622a575b1ea698ea18696 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 4 May 2021 11:20:17 -0700 Subject: [PATCH 070/197] DLPX-75450 [Backport of DLPX-75347 to 6.0.9.0] old rootfs versions are not removed on upgrade in 6.0.8.0 (#545) --- build.gradle | 1 + upgrade/README.md | 2 + upgrade/upgrade-scripts/rootfs-cleanup | 170 +++++++++++++++++++++++++ upgrade/upgrade-scripts/upgrade | 101 ++++++--------- 4 files changed, 215 insertions(+), 59 deletions(-) create mode 100755 upgrade/upgrade-scripts/rootfs-cleanup diff --git a/build.gradle b/build.gradle index 067c29dd..d7e6f6b2 100644 --- a/build.gradle +++ b/build.gradle @@ -90,6 +90,7 @@ def shellScripts = fileTree("scripts") + }) + fileTree("upgrade/upgrade-scripts", { exclude "README.md" + exclude "rootfs-cleanup" }) task shfmt(type: Exec) { diff --git a/upgrade/README.md b/upgrade/README.md index ba91d3db..c22ed943 100644 --- a/upgrade/README.md +++ b/upgrade/README.md @@ -39,6 +39,8 @@ Log into that VM using the "delphix" user, and run these commands: $ download-latest-image internal-dev $ sudo unpack-image internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v deferred + $ sudo /var/dlpx-update/latest/upgrade finalize + $ sudo /var/dlpx-update/latest/upgrade cleanup ## FAQ diff --git a/upgrade/upgrade-scripts/rootfs-cleanup b/upgrade/upgrade-scripts/rootfs-cleanup new file mode 100755 index 00000000..e2901241 --- /dev/null +++ b/upgrade/upgrade-scripts/rootfs-cleanup @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +import os +import re +import subprocess +from typing import Any, List, Type + +IMAGE_PATH = os.path.dirname(os.path.realpath(__file__)) + + +# +# Helper to compare two version strings using "dpkg --compare-versions". +# +def dpkgcmp(vers1: str, op: str, vers2: str) -> bool: + return subprocess.run(["dpkg", "--compare-versions", vers1, op, + vers2]).returncode == 0 + + +# +# Helper to obtain the "current-version" of a rootfs filesytem/snapshot. +# +def version(rootfs: str) -> str: + return str( + subprocess.check_output([ + "zfs", "get", "-Hpo", "value", "com.delphix:current-version", rootfs + ]).decode("UTF-8").strip()) + + +# +# Helper class used when sorting rootfs filesystems and snapshots. This allows +# us to use the "sorted" builtin, to build a list of all rootfs filesystems and +# snapshots on the system, sorted baseed on the "current-version" property. +# +class rootfscmp(object): + + def __init__(self, rootfs: str, *args: List[Any]): + self.rootfs = rootfs + + def __lt__(self, other: Type["rootfscmp"]) -> bool: + return dpkgcmp(version(self.rootfs), "lt", version(other.rootfs)) + + def __gt__(self, other: Type["rootfscmp"]) -> bool: + return dpkgcmp(version(self.rootfs), "gt", version(other.rootfs)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, rootfscmp): + return NotImplemented + return dpkgcmp(version(self.rootfs), "eq", version(other.rootfs)) + + def __le__(self, other: Type["rootfscmp"]) -> bool: + return dpkgcmp(version(self.rootfs), "le", version(other.rootfs)) + + def __ge__(self, other: Type["rootfscmp"]) -> bool: + return dpkgcmp(version(self.rootfs), "ge", version(other.rootfs)) + + def __ne__(self, other: object) -> bool: + if not isinstance(other, rootfscmp): + return NotImplemented + return dpkgcmp(version(self.rootfs), "ne", version(other.rootfs)) + + +def main() -> None: + snapshots = subprocess.check_output([ + "zfs", "list", "-r", "rpool/ROOT", "-d", "2", "-Ho", "name", "-t", + "snapshot" + ]).decode("UTF-8").splitlines() + + for snap in snapshots: + name = snap.split("@")[1] + + # + # We filter out any container-delphix snapshots, as those removed by + # the calls to "rootfs-container delete", on the relevant filesystems. + # + if re.search(r"^container-delphix.\w{7}$", name): + snapshots.remove(snap) + continue + + # + # Out of an abundance of caution, verify the snapshot name matches + # our expectations. + # + if not re.search(r"^execute-upgrade.\w{7}$", name): + raise SystemExit("unexpected snapshot name: '{}'".format(name)) + + filesystems = subprocess.check_output([ + "zfs", "list", "-r", "rpool/ROOT", "-d", "1", "-Ho", "name", "-t", + "filesystem" + ]).decode("UTF-8").splitlines() + + # + # The "rpool/ROOT" dataset is not versioned and also not eligible + # for deletion, so we must remove it from the list before we attempt + # to sort and remove rootfs snapshots and filesystems. + # + filesystems.remove('rpool/ROOT') + + current = subprocess.check_output(["get-appliance-version" + ]).decode("UTF-8").strip() + + # + # Our policy is to keep two delphix versions at any given time; the + # currently running version, and the version used prior to the + # current one. + # + # By sorting the list of all filesystems and snapshots on the sytem, + # we can easily determine the highest two versions on the system, + # and skip those versions (to maintain the policy described above); + # this is based on the assumption that the versions of an appliance + # will always increase (or remain constant, in the case of rollback) + # after an upgrade (i.e. we never upgrade to a lower version). + # + for rootfs in sorted(filesystems + snapshots, key=rootfscmp)[:-2]: + # + # In the event of a rollback, we want to be careful to not + # delete the currently running version, as well as any versions + # that may have failed to upgrade resulting in the rollback + # event to begin with. + # + # Additionally, we want to keep all rootfs snapshots that map to + # hotfixes for the currently running version. This way, we don't + # inadvertently remove a rootfs snapshot that we may need for + # root-causing a given problem; at least, not until we upgraded + # off of the problematic delphix version. + # + # Thus, to account for these cases, we skip any version that's + # equal to or greater than what's currently running. + # + # + if dpkgcmp(version(rootfs), "ge", current): + break + + # + # If a rollback event has occured, it's possible that we'll be + # unable to delete a snapshot or filesystem; e.g. if it's still + # being used as the "origin" of another filesystem. Rather than + # adding more complicated logic to handle this situation, we + # simply allow the commands to throw an error. + # + # If we fail to remove a snapshot or filesystem, this shouldn't + # negatively impact the appliance, apart from potentially + # consuming some more storage than strictly necessary. + # + # Further, as the appliance continues to be upgraded to new + # versions over time, these error conditions will subside; it + # just might take a couple upgrades to allow the previously + # "pinned" version to be removed and cleaned up. + # + # Lastly, it's possible for us to delete an entire rootfs + # container early in the iteration, and then attempt to delete + # some of that container's snapshots later in the iteration. + # Since deleting the container will either delete (or promote) + # all of that container's snapshots, the attempt (later) to + # delete the snapshots individually will silently fail. + # + if rootfs in filesystems: + subprocess.run([ + "{}/rootfs-container".format(IMAGE_PATH), "delete", + rootfs.split('/')[2] + ]) + elif rootfs in snapshots: + subprocess.run(["zfs", "destroy", "-r", rootfs]) + else: + raise SystemExit( + "rootfs not found in filesystems or snapshots: '{}'".format( + rootfs)) + + +if __name__ == "__main__": + main() diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 8973b027..ea95232a 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -361,6 +361,41 @@ function rollback() { } function finalize() { + source_upgrade_properties + + [[ -n "$UPGRADE_TYPE" ]] || + die "variable UPGRADE_TYPE is not set; is upgrade in progress?" + [[ -n "$UPGRADE_BASE_VERSION" ]] || + die "variable UPGRADE_BASE_VERSION is not set" + + case "$UPGRADE_TYPE" in + DEFERRED | FULL | ROLLBACK) ;; + *) + die "finalize is not supported for upgrade type: '$UPGRADE_TYPE'" + ;; + esac + + remove_upgrade_properties + + # + # We only want to remove this hotfix file after a successful + # upgrade attempt; at which point the old hotfix file will no + # longer be representative of the new rootfs contents. + # + # If a rollback occured, then we do not want to remove this + # file, since the currently running rootfs will be the "old" + # version, and still should match the hotfix file. We detect if + # a rollback occurred by the presence of this field (which would + # be set in the upgrade properites file). + # + if [[ -z "$ROLLBACK_BASE_VERSION" ]]; then + if [[ -f "$HOTFIX_PATH" ]]; then + rm -f "$HOTFIX_PATH" || die "failed to remove hotfix file" + fi + fi +} + +function cleanup() { if [[ "$DLPX_UPGRADE_DRY_RUN" == "true" ]]; then # # If we're executing a dry-run of the finalize @@ -388,67 +423,9 @@ function finalize() { die "failed to clean up illumos datasets" fi - # - # This first pass checks for available "execute.upgrade" snapshots - # and deletes snapshots that are older than the base upgrade version. - # This pass takes care of cleanup after in-place and deferred upgrades. - # - SNAPSHOT_LIST=$(zfs list -t snapshot -r rpool/ROOT -d 2 -Ho name) || - die "unable to determine available rollback snapshots" - - for snapshot in $SNAPSHOT_LIST; do - local SNAPSHOT_NAME - SNAPSHOT_NAME=$(echo "$snapshot" | awk -F@ '{print $2}') - # - # We skip container-delphix instances since `rootfs-container delete` - # handles them in the second pass below. - # - grep -qE "^container-delphix.[[:alnum:]]{7}$" <(echo "$SNAPSHOT_NAME") && - continue - grep -qE "^execute-upgrade.[[:alnum:]]{7}$" <(echo "$SNAPSHOT_NAME") || - die "unexpected snapshot name: '$snapshot'" - local SNAPSHOT_VERSION - SNAPSHOT_VERSION=$(zfs get -Hpo value "$PROP_CURRENT_VERSION" "$snapshot") || - die "failed to get snapshot version" - [[ "$SNAPSHOT_VERSION" != "-" ]] || - die "failed to get current version for snapshot '$snapshot'" - if compare_versions \ - "$SNAPSHOT_VERSION" "lt" "$UPGRADE_BASE_VERSION"; then - zfs destroy -r "$snapshot" || - die "failed to destroy rollback snapshot" - fi - done - - # - # This second pass checks for available filesystems and deletes those - # that are older than the base upgrade version. This pass takes care - # of cleanup following rollbacks and not-in-place upgrades. - # - FILESYSTEM_LIST=$(zfs list -t filesystem -r rpool/ROOT -d 1 -Ho name | tail -n +2) || - die "unable to determine available filesystems" - - for filesystem in $FILESYSTEM_LIST; do - local FILESYSTEM_VERSION - FILESYSTEM_VERSION=$(zfs get -Hpo value "$PROP_CURRENT_VERSION" "$filesystem") || - die "failed to get filesystem version" - [[ "$FILESYSTEM_VERSION" != "-" ]] || - die "failed to get current version for filesystem '$filesystem'" - if compare_versions \ - "$FILESYSTEM_VERSION" "lt" "$UPGRADE_BASE_VERSION"; then - local FILESYSTEM_NAME - FILESYSTEM_NAME=$(echo "$filesystem" | awk -F/ '{print $3}') - "$IMAGE_PATH/rootfs-container" delete "$FILESYSTEM_NAME" || - die "failed to delete filesystem '$filesystem" - fi - done + "$IMAGE_PATH/rootfs-cleanup" || die "failed to cleanup rootfs versions" rm -rf "$IMAGE_PATH" || die "failed to remove unpacked upgrade image" - - remove_upgrade_properties - - if [[ -f "$HOTFIX_PATH" ]]; then - rm -f "$HOTFIX_PATH" || die "failed to remove hotfix file" - fi } [[ "$EUID" -ne 0 ]] && die "must be run as root" @@ -507,6 +484,12 @@ finalize) shift 1 finalize "$@" ;; +cleanup) + verify_upgrade_not_in_progress + + shift 1 + cleanup "$@" + ;; *) usage "invalid option -- '$1'" ;; From 26c960f1697c1cef7d66bae8326670094b9e206d Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 5 May 2021 14:17:57 -0700 Subject: [PATCH 071/197] DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547) --- upgrade/upgrade-scripts/upgrade | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index ea95232a..5ab9add9 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -445,6 +445,15 @@ while getopts ':nv' c; do done shift $((OPTIND - 1)) +# +# Currently, we don't want to run upgrade verification for hotfixes, as +# we haven't yet implemented all the work necessary for the verification +# to behave properly for hotfixes. Thus, for now, if we detect this is a +# hotfix image, we disable upgrade verification. +# +source_version_information +[[ -n "$HOTFIX" ]] && DLPX_UPGRADE_SKIP_VERIFY="true" + case "$1" in deferred) verify_upgrade_not_in_progress From f6421aa1f2112ec17eb53737d079ff87f1e1a533 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 20 May 2021 15:30:04 -0700 Subject: [PATCH 072/197] DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549) --- upgrade/upgrade-scripts/execute | 68 +++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 2ff40480..38b105da 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -242,19 +242,32 @@ dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || # # While we marked all packages to "auto" above, so they'd be removed -# when they're no longer required, we want to prevent the removal of old -# kernel packages. This is so that we can boot into the old kernel(s) -# if we ever need to; e.g. after a failed kernel upgrade. We accomplish -# this by marking all "delphix-kernel-*" packages as "manual"-ly -# installed. The "delphix-kernel-*" package(s) will have package -# dependencies on the linux kernel package(s), and all other kernel -# packages that're required for that specific kernel (e.g. zfs kernel -# module package, connstat kernel module package, etc.). Thus, as long -# as the "delphix-kernel-*" package remains installed, all other -# dependent packages should also remain installed. -# -dpkg-query -Wf '${Package}\n' "delphix-kernel-*" | xargs apt-mark manual || - die "failed to mark 'delphix-kernel-*' packages as 'manual' installed" +# when they're no longer required, we treat kernel packages slightly +# differently. For disaster recovery type situations (e.g. the new +# kernel fails to boot), we always want to keep a "known good" kernel +# installed on the system. This way, if we have problems with the new +# kernel installed as part of this upgrade, we'll hopefully have the old +# one still available, to help recover us out of a failed upgrade. +# +# Thus, the logic here ensures that the currently running kernel is not +# removed when we run the "apt-get autoremove" logic later in this +# script. Further, we only need to preserve the "delphix-kernel" package +# here, as this will preserve all other packages it depends on; e.g. all +# kernel module packages, etc. +# +# Lastly, we use the currently running kernel as the "known good" +# version that we want to preserve, since the fact that we're currently +# running on it means it's been verified to (at the very least) boot up +# properly. We don't track which of the installed kernels have been +# verified to work (e.g. booted from), and due to deferred upgrades, +# it's possible to install new kernels and never actually use them. +# Thus, of all the currently installed kernels, the only one that we +# know to function properly, is the currently running one; which is why +# we explicitly choose to keep the currently running version. +# +dpkg-query -Wf '${Package}\n' "delphix-kernel-$(uname -r)" | + xargs apt-mark manual || + die "failed to mark delphix-kernel packages as 'manual' installed" # # In order to perform an upgrade of the Delphix appliance, we must first @@ -318,8 +331,33 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # all packages that are no longer required. For example, if the old # "delphix-entire" version required a package, and the new version no # longer requires that package, the commands run above will not remove -# the package. Thus, we need to run "autoremove" to ensure all packages -# that fall into this category are removed. +# the package. +# +# First, since "apt-get autoremove" does not work properly for the +# kernel packages, we have to handle the removal of those here. +# +KVERS=$(apt-mark showauto 'delphix-kernel-.*' | sed 's/^delphix-kernel-//') || + die "failed to determine list of kernel versions to remove" +for KVER in $KVERS; do + apt_get purge -y "delphix-kernel-$KVER" "linux-image-$KVER" || + die "failed to remove kernel packages for version '$KVER'" +done + +# +# Next, APT will automatically try to prevent kernel packages from being +# removed via "apt-get autoremove", in an attempt to be "helpful". Since +# we're already taking the necessary precautions that make sense for our +# appliance w.r.t. kernel packages, we need to disable those native APT +# protections by removing this "01autoremove" file. +# +rm -rf /etc/apt/apt.conf.d/01autoremove || + die "failed to remove '/etc/apt/apt.conf.d/01autoremove' file" + +# +# Finally, now that we've handled the two complications above, we can +# use "apt-get autoremove" to properly remove all old packages; i.e. +# ones that are no longer needed on the currently installed version of +# the appliance. # apt_get autoremove --purge -y || die "autoremove after upgrade failed" From b5639313c11bdd29fd43861b948780a510b73a86 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 2 Jun 2021 12:41:09 -0700 Subject: [PATCH 073/197] Don't remove "delphix-entire" when generating image diffs (#536) (#537) Currently, if we generate an upgrade image diff from two images that have the same delphix-entire package version, the resultant Aptly repository will not contain any delphix-entire package in it. This is problematic when that Aptly repository is later used with the script used to generate an upgrade image from that repository, as the script requires the repository to contain a "delphix-entire" package in it. Thus, to avoid this issue, this change modifies the image diff logic to ensure we never remove the "delphix-entire" package. --- scripts/aptly-repo-from-image-diff.sh | 30 ++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/aptly-repo-from-image-diff.sh b/scripts/aptly-repo-from-image-diff.sh index 826d503c..b987abb3 100755 --- a/scripts/aptly-repo-from-image-diff.sh +++ b/scripts/aptly-repo-from-image-diff.sh @@ -100,14 +100,34 @@ aptly repo search image-a | xargs aptly repo copy image-a upgrade-repository || # # Here we're performing step 3 from the comment above, but since the -# "delphix-upgrade-verification" package is a bit different than most -# other packages on a Delphix appliance, we need to handle that package -# uniquely here. Specifically, we want to ensure this package is always -# contained in the resultant aptly repository (even if the package is -# the same within both image A and image B). +# "delphix-upgrade-verification" and "delphix-entire" packages are a bit +# different than most other packages on a Delphix appliance, we need to +# handle these packages uniquely here. Specifically, we want to ensure these +# packages are always contained in the resultant Aptly repository (even if +# the package is the same within both image A and image B), as each package +# is essential to the proper functioning of any Delphix appliance upgrade. +# +# The "delphix-upgrade-verification" package is used to perform upgrade +# specific logic (and verification) during the upgrade process, such that we +# can avoid common pitfalls that would result in the upgrade process +# failing. Without this package being contained in the repository (and thus, +# any upgrade image generated from the repository), the verification stage +# of upgrade would fail, leading to the entire upgrade process failing. For +# more details, see the "verify-jar" script within the "upgrade-scripts" +# directory. +# +# Additionally, the "delphix-entire" package is used by the upgrade logic to +# determine which packages constitute the Delphix version the appliance is +# transitioning to, and ensure all of those packages are upgraded (or +# downgraded) to the correct versions based on the information provided by +# this package. Without this package, the upgrade logic would fail, as it +# would not be able to determine what packages to upgrade (or downgrade). +# For more details, see the "execute" script within the "upgrade-scripts" +# directory. # aptly repo search image-b | grep -v "^delphix-upgrade-verification" | + grep -v "^delphix-entire" | xargs aptly repo remove upgrade-repository || die "failed to remove packages from repository: 'upgrade-repository'" From c09974e63c2e047dc44d1b7fff8787d73d96b739 Mon Sep 17 00:00:00 2001 From: Matt Skinner Date: Mon, 7 Jun 2021 17:49:33 -0400 Subject: [PATCH 074/197] DLPX-75091 DLPX-75122 Make upgrade image prepare script hotfix aware (#543) (#550) DLPX-75091 Make upgrade image prepare script hotfix aware DLPX-75122 Use delphix version and hotfix in version.info in verify-jar script --- upgrade/prepare | 5 ++++- upgrade/upgrade-scripts/common.sh | 11 ++--------- upgrade/upgrade-scripts/upgrade | 6 +++--- upgrade/upgrade-scripts/verify-jar | 13 ++++++++++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/upgrade/prepare b/upgrade/prepare index a6641920..793cfe40 100755 --- a/upgrade/prepare +++ b/upgrade/prepare @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - UPDATE_DIR=${UPDATE_DIR:-/var/dlpx-update} function die() { @@ -107,6 +106,10 @@ fi popd &>/dev/null || die "'popd' failed" +if [[ -n "$HOTFIX" ]]; then + VERSION="$VERSION-$HOTFIX" +fi + $opt_f && rm -rf "${UPDATE_DIR:?}/$VERSION" >/dev/null 2>&1 [[ -d "$UPDATE_DIR/$VERSION" ]] && die 18 "version $VERSION already exists" diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index b917a220..af027af8 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -141,10 +141,6 @@ function get_image_path() { readlink -f "${BASH_SOURCE%/*}" } -function get_image_version() { - basename "$(get_image_path)" -} - function get_mounted_rootfs_container_dataset() { dirname "$(zfs list -Hpo name /)" } @@ -256,13 +252,10 @@ function source_version_information() { local IMAGE_PATH="${IMAGE_PATH:-$(get_image_path)}" [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" - local IMAGE_VERSION="${IMAGE_VERSION:-$(get_image_version)}" - [[ -n "$IMAGE_VERSION" ]] || die "failed to determine image version" - [[ -f "$IMAGE_PATH/version.info" ]] || - die "image for version '$IMAGE_VERSION' missing version.info" + die "image missing version.info for $IMAGE_PATH" . "$IMAGE_PATH/version.info" || - die "failed to source version.info for version '$IMAGE_VERSION'" + die "failed to source version.info for $IMAGE_PATH" [[ -n "$VERSION" ]] || die "VERSION is empty" [[ -n "$MINIMUM_VERSION" ]] || die "MINIMUM_VERSION is empty" diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 5ab9add9..ab7d8d94 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -20,9 +20,6 @@ IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" -IMAGE_VERSION=$(get_image_version) -[[ -n "$IMAGE_VERSION" ]] || die "failed to determine image version" - # # This variable is used to determine if a "dry run" of the upgrade # should be performed, as opposed to an actual upgrade of the system. @@ -320,11 +317,14 @@ function rollback() { ROLLBACK_BASE_VERSION="$(get_current_version)" [[ -n "$ROLLBACK_BASE_VERSION" ]] || die "unable to determine current appliance version" + ROLLBACK_BASE_HOTFIX="$(get_current_hotfix)" set_upgrade_property "ROLLBACK_BASE_CONTAINER" "$ROLLBACK_BASE_CONTAINER" || die "failed setting 'ROLLBACK_BASE_CONTAINER' property" set_upgrade_property "ROLLBACK_BASE_VERSION" "$ROLLBACK_BASE_VERSION" || die "failed setting 'ROLLBACK_BASE_VERSION' property" + set_upgrade_property "ROLLBACK_BASE_HOTFIX" "$ROLLBACK_BASE_HOTFIX" || + die "failed setting 'ROLLBACK_BASE_HOTFIX' property" if [[ "$UPGRADE_BASE_CONTAINER" == "$ROLLBACK_BASE_CONTAINER" ]]; then # diff --git a/upgrade/upgrade-scripts/verify-jar b/upgrade/upgrade-scripts/verify-jar index 7faa4b8b..43d6cefa 100755 --- a/upgrade/upgrade-scripts/verify-jar +++ b/upgrade/upgrade-scripts/verify-jar @@ -18,8 +18,14 @@ # shellcheck disable=SC1090 . "${BASH_SOURCE%/*}/common.sh" -IMAGE_VERSION=$(get_image_version) -[[ -n "$IMAGE_VERSION" ]] || die "failed to determine image version" +# We source the version information to bring in values from +# the version.info file, such as +# - VERSION : the appliance version +# - HOTFIX : the hotfix version if any + +source_version_information + +[[ -n "$VERSION" ]] || die "failed to determine image version" IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" @@ -182,7 +188,8 @@ VERIFY_OPTIONS=( "-d" "${opt_o:-${LOG_DIRECTORY}/${CONTAINER}/upgrade_verify.json}" "-f" "${opt_f:-1}" "-l" "${opt_l:-en-US}" - "-v" "$IMAGE_VERSION" + "-v" "$VERSION" + "-h" "$HOTFIX" "-pl" "25" "-ph" "80" ) From 0a7459d2b77a2de58780889983e43f3e35cbec2d Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 10 Jun 2021 09:30:28 -0700 Subject: [PATCH 075/197] Revert "DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549)" (#555) This reverts commit f6421aa1f2112ec17eb53737d079ff87f1e1a533. --- upgrade/upgrade-scripts/execute | 68 ++++++++------------------------- 1 file changed, 15 insertions(+), 53 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 38b105da..2ff40480 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -242,32 +242,19 @@ dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || # # While we marked all packages to "auto" above, so they'd be removed -# when they're no longer required, we treat kernel packages slightly -# differently. For disaster recovery type situations (e.g. the new -# kernel fails to boot), we always want to keep a "known good" kernel -# installed on the system. This way, if we have problems with the new -# kernel installed as part of this upgrade, we'll hopefully have the old -# one still available, to help recover us out of a failed upgrade. -# -# Thus, the logic here ensures that the currently running kernel is not -# removed when we run the "apt-get autoremove" logic later in this -# script. Further, we only need to preserve the "delphix-kernel" package -# here, as this will preserve all other packages it depends on; e.g. all -# kernel module packages, etc. -# -# Lastly, we use the currently running kernel as the "known good" -# version that we want to preserve, since the fact that we're currently -# running on it means it's been verified to (at the very least) boot up -# properly. We don't track which of the installed kernels have been -# verified to work (e.g. booted from), and due to deferred upgrades, -# it's possible to install new kernels and never actually use them. -# Thus, of all the currently installed kernels, the only one that we -# know to function properly, is the currently running one; which is why -# we explicitly choose to keep the currently running version. -# -dpkg-query -Wf '${Package}\n' "delphix-kernel-$(uname -r)" | - xargs apt-mark manual || - die "failed to mark delphix-kernel packages as 'manual' installed" +# when they're no longer required, we want to prevent the removal of old +# kernel packages. This is so that we can boot into the old kernel(s) +# if we ever need to; e.g. after a failed kernel upgrade. We accomplish +# this by marking all "delphix-kernel-*" packages as "manual"-ly +# installed. The "delphix-kernel-*" package(s) will have package +# dependencies on the linux kernel package(s), and all other kernel +# packages that're required for that specific kernel (e.g. zfs kernel +# module package, connstat kernel module package, etc.). Thus, as long +# as the "delphix-kernel-*" package remains installed, all other +# dependent packages should also remain installed. +# +dpkg-query -Wf '${Package}\n' "delphix-kernel-*" | xargs apt-mark manual || + die "failed to mark 'delphix-kernel-*' packages as 'manual' installed" # # In order to perform an upgrade of the Delphix appliance, we must first @@ -331,33 +318,8 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # all packages that are no longer required. For example, if the old # "delphix-entire" version required a package, and the new version no # longer requires that package, the commands run above will not remove -# the package. -# -# First, since "apt-get autoremove" does not work properly for the -# kernel packages, we have to handle the removal of those here. -# -KVERS=$(apt-mark showauto 'delphix-kernel-.*' | sed 's/^delphix-kernel-//') || - die "failed to determine list of kernel versions to remove" -for KVER in $KVERS; do - apt_get purge -y "delphix-kernel-$KVER" "linux-image-$KVER" || - die "failed to remove kernel packages for version '$KVER'" -done - -# -# Next, APT will automatically try to prevent kernel packages from being -# removed via "apt-get autoremove", in an attempt to be "helpful". Since -# we're already taking the necessary precautions that make sense for our -# appliance w.r.t. kernel packages, we need to disable those native APT -# protections by removing this "01autoremove" file. -# -rm -rf /etc/apt/apt.conf.d/01autoremove || - die "failed to remove '/etc/apt/apt.conf.d/01autoremove' file" - -# -# Finally, now that we've handled the two complications above, we can -# use "apt-get autoremove" to properly remove all old packages; i.e. -# ones that are no longer needed on the currently installed version of -# the appliance. +# the package. Thus, we need to run "autoremove" to ensure all packages +# that fall into this category are removed. # apt_get autoremove --purge -y || die "autoremove after upgrade failed" From c232712e895abc32e3530a9e0a4f1d4a10a109d3 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 10 Jun 2021 09:34:53 -0700 Subject: [PATCH 076/197] Revert "DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547)" (#556) This reverts commit 26c960f1697c1cef7d66bae8326670094b9e206d. --- upgrade/upgrade-scripts/upgrade | 9 --------- 1 file changed, 9 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index ab7d8d94..f2e544e9 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -445,15 +445,6 @@ while getopts ':nv' c; do done shift $((OPTIND - 1)) -# -# Currently, we don't want to run upgrade verification for hotfixes, as -# we haven't yet implemented all the work necessary for the verification -# to behave properly for hotfixes. Thus, for now, if we detect this is a -# hotfix image, we disable upgrade verification. -# -source_version_information -[[ -n "$HOTFIX" ]] && DLPX_UPGRADE_SKIP_VERIFY="true" - case "$1" in deferred) verify_upgrade_not_in_progress From beb73b82e48a30c0df1615eba02850e3ec774a8d Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 22 Jun 2021 11:52:04 -0700 Subject: [PATCH 077/197] DLPX-76243 management stack failed to come back after rollback upgrade (#558) --- upgrade/upgrade-scripts/upgrade | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index f2e544e9..97448865 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -323,8 +323,11 @@ function rollback() { die "failed setting 'ROLLBACK_BASE_CONTAINER' property" set_upgrade_property "ROLLBACK_BASE_VERSION" "$ROLLBACK_BASE_VERSION" || die "failed setting 'ROLLBACK_BASE_VERSION' property" - set_upgrade_property "ROLLBACK_BASE_HOTFIX" "$ROLLBACK_BASE_HOTFIX" || - die "failed setting 'ROLLBACK_BASE_HOTFIX' property" + + if [[ -n "$ROLLBACK_BASE_HOTFIX" ]]; then + set_upgrade_property "ROLLBACK_BASE_HOTFIX" "$ROLLBACK_BASE_HOTFIX" || + die "failed setting 'ROLLBACK_BASE_HOTFIX' property" + fi if [[ "$UPGRADE_BASE_CONTAINER" == "$ROLLBACK_BASE_CONTAINER" ]]; then # From 62def8aa00693bebae06819fdb00e0ffb5d01a19 Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Wed, 23 Jun 2021 09:15:25 -0700 Subject: [PATCH 078/197] CP-5356 [Backport of CP-5334 to 6.0.9.0] Add ufw package to appliance-build (#559) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 4de3b32a..e8b13d51 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -1,5 +1,5 @@ # -# Copyright 2018 Delphix +# Copyright 2018, 2021 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ name: - nftables - snmptrapd + - ufw state: present register: result until: result is not failed From a36cbc2813b8f163ac3ae9dc91fbf87a850ac4c5 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 24 Jun 2021 17:53:00 -0700 Subject: [PATCH 079/197] DLPX-76329 [Backport of DLPX-76328 to 6.0.9.0] upgrade script obtains hotfix version incorrectly (#564) In commit e7dd85e2 we introduced the `get_hotfix_version` function for obtaining the hotfix version of the currently running root filesystem. The problem is that in c4357a4e we added logic that intended to use this function, but instead tries to call the `get_current_hotfix` function, which doesn't exist. This leads us to never retreiving the hotfix version correctly via the `rollback` subcommand of the `upgrade` script. --- upgrade/upgrade-scripts/upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 97448865..840ce21e 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -317,7 +317,7 @@ function rollback() { ROLLBACK_BASE_VERSION="$(get_current_version)" [[ -n "$ROLLBACK_BASE_VERSION" ]] || die "unable to determine current appliance version" - ROLLBACK_BASE_HOTFIX="$(get_current_hotfix)" + ROLLBACK_BASE_HOTFIX="$(get_hotfix_version)" set_upgrade_property "ROLLBACK_BASE_CONTAINER" "$ROLLBACK_BASE_CONTAINER" || die "failed setting 'ROLLBACK_BASE_CONTAINER' property" From f1643696d6c3a653693333169977dbb1ab19e296 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 29 Jun 2021 14:25:38 -0700 Subject: [PATCH 080/197] DLPX-74045 adapt upgrade "execute" script to better handle new hotfix workflow (#520) (#568) Currently when applying a hotfix image generated by the new hotfix workflow being developed in CP-4190, nothing will happen when the upgrade is attempted. This is due to the "delphix-entire" package in the hotfix being installed, matching the version of the package that's already installed. Thus, the package manager chooses not to install the new packagecontained in the upgrade/hotfix image, since it thinks the same package is already installed, due to the two having the same version. We need to adapt the "execute" script that's used during upgrade, such that we always install the new "delphix-entire" package contained in the upgrade/hotfix image, even if the package versions are identical. --- upgrade/upgrade-scripts/execute | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 2ff40480..16e817d4 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -303,9 +303,23 @@ dpkg-query -Wf '${Package}\n' "delphix-kernel-*" | xargs apt-mark manual || # # shellcheck disable=SC2153 -apt_get install -y --allow-downgrades "delphix-entire-$platform=$VERSION" || +apt_get install \ + -y --allow-downgrades --reinstall \ + "delphix-entire-$platform=$VERSION" || die "upgrade failed; from '$CURRENT_VERSION' to '$VERSION'" +# +# Since we mark all currently installed packages as "auto" earlier in +# this script, we need to ensure the delphix-entire package that we just +# installed is marked "manual", to avoid it being removed later in this +# script. We must explicitly do this here, as it seems this isn't +# automatially done when installing or upgrading the package; +# particularly in the case of "--reinstall", which replaces the current +# package with a new package of the same version (i.e. for hotfixes). +# +apt-mark manual "delphix-entire-$platform" || + die "failed to mark 'delphix-entire' package as 'manual' installed" + [[ -f "/usr/share/doc/delphix-entire-$platform/packages.list.gz" ]] || die "delphix-entire's packages.list.gz file is missing" From fd57669ffb6f957fc2621d228f1093968c3f7aa9 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 6 Jul 2021 12:59:55 -0700 Subject: [PATCH 081/197] TOOL-11824 Harden against mistakes when running via Jenkins (#569) (#570) This change adds logic to cause a failure when certain environment variables are missing, and the build is being run via Jenkins. When environment variables are missing, we currently will provide default values, such that the build doesn't fail. These default values generally are not sufficient for anything besides allowing the build to complete (e.g. the resultant upgrade images may fail upgrade). Further, the main motivation for generating these default values, is to make it easier to run the build manually, and aid developer iteration. Thus, when it's clear that the build is being run via Jenkins, where these missing environment variables more likely point to an error or broken automation, than an intentional omission of these variables, it's better to report the error than set and use a default value. This way, we can more quickly detect and address the problem, than silently produce (potentially) incorrect or broken artifacts. --- scripts/upgrade-image-from-aptly-repo.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/upgrade-image-from-aptly-repo.sh b/scripts/upgrade-image-from-aptly-repo.sh index 7eb18aa1..54358647 100755 --- a/scripts/upgrade-image-from-aptly-repo.sh +++ b/scripts/upgrade-image-from-aptly-repo.sh @@ -122,6 +122,8 @@ if [[ -n "$DELPHIX_UPGRADE_MINIMUM_VERSION" ]]; then "s/@@MINIMUM_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_VERSION/" \ version.info || die "failed to set MINIMUM_VERSION in version.info" +elif [[ -n "$JENKINS_URL" ]]; then + die "DELPHIX_UPGRADE_MINIMUM_VERSION not specified" else sed -i "s/@@MINIMUM_VERSION@@/0.0.0.0/" version.info || die "failed to set MINIMUM_VERSION in version.info" @@ -132,6 +134,8 @@ if [[ -n "$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION" ]]; then "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION/" \ version.info || die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" +elif [[ -n "$JENKINS_URL" ]]; then + die "DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION not specified" else sed -i "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/0.0.0.0/" version.info || die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" @@ -147,6 +151,12 @@ sha256sum payload.tar.gz version.info verification-version.info prepare >SHA256S # DELPHIX_SIGNATURE_TOKEN environment variable contents to stdout. # set +o xtrace +if [[ -n "$JENKINS_URL" ]]; then + [[ -n "$DELPHIX_SIGNATURE_VERSIONS" ]] || die "DELPHIX_SIGNATURE_VERSIONS not specified" + [[ -n "$DELPHIX_SIGNATURE_TOKEN" ]] || die "DELPHIX_SIGNATURE_TOKEN not specified" + [[ -n "$DELPHIX_SIGNATURE_URL" ]] || die "DELPHIX_SIGNATURE_URL not specified" +fi + if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then echo "{\"data\": \"$(base64 -w 0 SHA256SUMS)\"}" >sign-request.payload || die "failed to generate sigh-request.payload file" From b7170df2ef27af205659c2206dbb315f28289b1c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 9 Jul 2021 10:57:41 -0400 Subject: [PATCH 082/197] [Backport to 6.0] Remove code for building migration images (#571) Co-authored-by: John Gallagher --- README.md | 1 - build.gradle | 3 - live-build/build.gradle | 8 - .../vm-artifacts/84-clear-apt-cache.binary | 4 +- .../90-linux-migration-artifact.binary | 183 ------- live-build/misc/migration-scripts/dx_apply | 362 ------------- live-build/misc/migration-scripts/dx_delete | 122 ----- live-build/misc/migration-scripts/dx_execute | 237 --------- live-build/misc/migration-scripts/dx_prepare | 23 - .../migration-scripts/dx_upg_pause_options | 217 -------- .../migration-scripts/dx_upg_stress_options | 480 ------------------ live-build/misc/migration-scripts/dx_verify | 211 -------- .../migration-scripts/etc_system_whitelist | 10 - scripts/run-live-build.sh | 3 +- 14 files changed, 3 insertions(+), 1861 deletions(-) delete mode 100755 live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary delete mode 100755 live-build/misc/migration-scripts/dx_apply delete mode 100755 live-build/misc/migration-scripts/dx_delete delete mode 100755 live-build/misc/migration-scripts/dx_execute delete mode 100755 live-build/misc/migration-scripts/dx_prepare delete mode 100755 live-build/misc/migration-scripts/dx_upg_pause_options delete mode 100755 live-build/misc/migration-scripts/dx_upg_stress_options delete mode 100755 live-build/misc/migration-scripts/dx_verify delete mode 100644 live-build/misc/migration-scripts/etc_system_whitelist diff --git a/README.md b/README.md index a22bdd99..6875d81a 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,6 @@ the "live-build/build/artifacts/" directory: $ ls -lh live-build/build/artifacts/ total 1.9G -rw-r--r-- 1 root root 275M Jan 11 22:31 internal-minimal-kvm.debs.tar.gz - -rw-r--r-- 1 root root 45 Jan 11 22:31 internal-minimal-kvm.migration.tar.gz -rw-r--r-- 1 root root 636M Jan 11 22:33 internal-minimal-kvm.qcow2 The appliance produced will contain a kernel optimized for the diff --git a/build.gradle b/build.gradle index d7e6f6b2..85e0bcc5 100644 --- a/build.gradle +++ b/build.gradle @@ -82,9 +82,6 @@ def shellScripts = fileTree("scripts") + fileTree("live-build/config/hooks").include({ details -> details.file.canExecute() }) + - fileTree("live-build/misc/migration-scripts", { - exclude "etc_system_whitelist" - }) + fileTree("upgrade", { include "prepare" }) + diff --git a/live-build/build.gradle b/live-build/build.gradle index 89717ece..4443c282 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -125,18 +125,10 @@ for (variant in allVariants) { outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" break case vmArtifactsRunType: - if (variant != "internal-minimal") { - // We don't bother creating a migration image for internal-minimal - outputs.file "${buildDir}/artifacts/${variant}-${platform}.migration.tar.gz" - } outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" break case allRunType: outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" - if (variant != "internal-minimal") { - // We don't bother creating a migration image for internal-minimal - outputs.file "${buildDir}/artifacts/${variant}-${platform}.migration.tar.gz" - } outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" break } diff --git a/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary b/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary index 0ffb3045..2e79f5d2 100755 --- a/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary +++ b/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary @@ -16,8 +16,8 @@ # # -# To reduce the size of our VM and migration images, we clean the APT -# cache, and remove its "lists" files. +# To reduce the size of our VM images, we clean the APT cache and remove its +# "lists" files. # chroot binary apt-get clean chroot binary rm -rf /var/lib/apt/lists/* diff --git a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary deleted file mode 100755 index 269a508f..00000000 --- a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/bash -eux -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is intended to be used as part of Delphix's build process. -# Its role is to convert the "binary" directory generated by live-build, -# into a cpio archive that will be wrapper in a tarball. That tarball is -# the migration artifact that illumos-based engines use to migrate to -# Linux. -# -MIGRATION_ARTIFACT="$ARTIFACT_NAME.migration.tar.gz" - -# -# At the time of this writing dx_unpack.sh expects a tarball of the -# following structure: -# -# depot/ the top level container -# depot/dx_prepare top-level script to prepare the archive -# for upgrade -# depot/version.info version and date information about the -# upgrade image -# depot/hashes the sha256sum of any upgrade artifacts -# (see below) -# depot/hashes.sig[.] one cryptographic signature of depot/hashes -# per Delphix release -# depot/ generally cpio archives of OS and appliance -# -# As dx_upack.sh is the only piece of code that runs from the old -# world, we treat it as a contract that lists the bare minimum -# requirements for our artifact. -# -DEPOT_DIRECTORY=depot - -# -# Delete any leftover files from previous runs. -# -rm -rf "$MIGRATION_ARTIFACT" "$DEPOT_DIRECTORY" - -# -# Not all variants include the virtualization application (e.g. -# internal-minimal) so don't bother generating a migration image -# for those. /opt/delphix is always created as part of the -# appliance. Thus, checking if it exists is a quick way for us to -# distinguish between variants that have the appliance from -# the ones that don't. -# -if [[ ! -d binary/opt/delphix ]]; then - echo "This artifact does not contain the appliance." \ - " Skipping migration image..." - exit 0 -fi - -mkdir $DEPOT_DIRECTORY - -# -# Generate archive for os root directory. -# -# Note: We temporarily change directory in the subshell (we move to the -# "binary" directory) because find prints everything as a relative -# path to the current directory and we don't want the generated archive -# to have "binary" as its top-level directory. -# -( - cd binary - find . -print | cpio -oc -) >$DEPOT_DIRECTORY/os-root.cpio - -# -# Copy all migration scripts including dx_prepare which is needed by -# dx_unpack. -# -cp migration-scripts/* $DEPOT_DIRECTORY - -test -n "$DELPHIX_APPLIANCE_VERSION" - -# -# There may be a version.info file in the current directory already. -# That version.info is Linux-specific and does not necessarily have -# all the fields that the illumos upgrade process requires. Thus, we -# create our own special version info directly in the depot directory -# with all the fields expected by dx_unpack. -# -{ - # - # Note that both the following field is bogus and supplied - # solely for making dx_unpack happy. - # - echo "DLPX_OS_VERSION=Linux" - - # - # UpgradeManager expects a valid date for DLPX_DATE. - # - echo "DLPX_DATE=$(date '+%Y.%m.%d.%H.%M.%S')" - - # - # DLPX_MIN_VERSION is specified for ensuring that we are - # migrating from the right version. - # - # Note that this will be checked by the dx_apply script - # of migration, so every time we bump this number we - # need to bump the one from the checks there too. - # - echo "DLPX_MIN_VERSION=5.3.6.0" - - # - # DLPX_VERSION is set explicitly to match the version of the - # virtualization appliance, which is what upgrades in Linux - # do and therefore we try to stay consistent as this field - # really can be anything (e.g. a date, a point-version, a - # nickname, etc.) - # - # shellcheck disable=SC2016 - echo "DLPX_VERSION=$DELPHIX_APPLIANCE_VERSION" -} >$DEPOT_DIRECTORY/version.info - -# -# Generate hashes file. -# -( - cd binary - find . -type f -print0 | xargs -0 sha256sum -) >$DEPOT_DIRECTORY/os-root.hashes -( - cd $DEPOT_DIRECTORY - sha256sum ./* >hashes -) - -# -# DELPHIX_SIGNATURE_TOKEN is used for signing our migration image. -# The signature is later unpacked and verified on the VM -# performing the migration. If DELPHIX_SIGNATURE_TOKEN has not been -# specified by the user when running this hook we skip generating -# the signature for this image. Similarly for DELPHIX_SIGNATURE_URL. -# -if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then - # - # Generate depot/hashes.sig[.] - # - # Assumption: we always migrate from version 5.3 - # - VERSION="5.3" - SIGN_URL="$DELPHIX_SIGNATURE_URL/upgrade/keyVersion/$VERSION/sign" - ( - cd $DEPOT_DIRECTORY - - # Encode payload for signature request - base64 -w 0 hashes >hashes64 - echo "{\"data\": \"$(cat hashes64)\"}" >hashes64.payload - - # Request signature - curl -s -S -f -H "Content-Type: application/json" \ - -u "$DELPHIX_SIGNATURE_TOKEN" "$SIGN_URL" -d @hashes64.payload >hashes.response - - # Decode generated signature - jq -r .signature sig.encoded - base64 -d sig.encoded >hashes.sig.$VERSION - - # Remove intermediate files - rm hashes64 hashes64.payload hashes.response sig.encoded - ) -fi - -# -# Tape-ARchive & Compress Zee File! -# -# We enable verbose output in the hope that all the files that are part of -# the artifact are logged somewhere which could aid debugging in the future. -# -tar -cvf - $DEPOT_DIRECTORY | pigz >"$MIGRATION_ARTIFACT" diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply deleted file mode 100755 index c02a9df9..00000000 --- a/live-build/misc/migration-scripts/dx_apply +++ /dev/null @@ -1,362 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Creates the dataset layout/hierarchy expected by linux-upgrade for the -# archive from the migration image to be unpacked. It also takes the -# required steps and adds an entry to the FreeBSD bootloader so we can -# optionally boot into Linux. -# - -set -o pipefail - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -function usage() { - echo "$(basename "$0"): $*" >&2 - echo "Usage: $(basename "$0") " - exit 2 -} - -function report_progress_inc() { - echo "Progress increment: $(date +%T:%N%z), $1, $2" -} - -[[ "$(uname -s)" == "SunOS" ]] || die "script can only be used in illumos" - -while getopts ':h' opt; do - case $opt in - h) usage ;; - *) usage "Invalid option: -$OPTARG." ;; - esac -done - -ARCHIVE_DIR="$1" -[[ -n $ARCHIVE_DIR ]] || usage -[[ -d $ARCHIVE_DIR ]] || die "$ARCHIVE_DIR is not a directory" -[[ -f $ARCHIVE_DIR/version.info ]] || - die "$ARCHIVE_DIR does not have a version.info file" - -. "$ARCHIVE_DIR/version.info" - -# -# The root filesystem container needs to have the appliance version -# embedded in it as a dataset property, thus if this version was not -# found in the upgrade image, we need to error out (preferably before -# we do anything else, to avoid having to do cleanup later). -# -[[ -n "$DLPX_VERSION" ]] || die "DLPX_VERSION variable is missing" - -MIN_MIGRATION_VERSION="5.3.6.0" -[[ "$DLPX_MIN_VERSION" == "$MIN_MIGRATION_VERSION" ]] || - die "expected DLPX_MIN_VERSION for migration to be" \ - "$MIN_MIGRATION_VERSION" - -# -# Ensure that this VM is version 5.3.5.X or greater (but not 5.4 or 6.X). -# -CURRENT_DDS=$(dirname "$(mount | awk '/^\/opt\/delphix /{ print $3 }')") -CURRENT_VERSION=$(basename "$CURRENT_DDS") -MAJOR_VERSION_0=$(echo "$CURRENT_VERSION" | cut -d. -f1) -MAJOR_VERSION_1=$(echo "$CURRENT_VERSION" | cut -d. -f2) -[[ $MAJOR_VERSION_0 -eq 5 ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" -[[ $MAJOR_VERSION_1 -eq 3 ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" -MINOR_VERSION=$(echo "$CURRENT_VERSION" | cut -d. -f3) -MIGRATION_MINOR_VERSION=$(echo "$MIN_MIGRATION_VERSION" | cut -d. -f3) -[[ $MINOR_VERSION -ge $MIGRATION_MINOR_VERSION ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" - -DX_UPG_STRESS=$ARCHIVE_DIR/dx_upg_stress_options -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source -__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_AFTER_VERSION_CHECK" - -# -# Get the root dataset and the current ZFS pool that we're currently using. -# -RDS=$(mount | awk '/^\/ /{ print $3 }') -RPOOL=${RDS%%/*} - -# -# Cleanup any previous intermediate state. -# -"${BASH_SOURCE%/*}/dx_delete" || - die "failed to cleanup previous state with dx_delete" - -# -# Save a copy of the boot menu to restore if we aren't upgrading -# -cp "/boot/menu.rc.local" "/boot/menu.rc.local.copy" || - die "failed to save copy of /boot/menu.rc.local" - -report_progress_inc 20 - -# -# Create dataset layout similar to the linux upgrade. -# -TMPDIR=$(mktemp -d -p "/tmp" -t delphix.XXXXXXX) -FSNAME=$(basename "$TMPDIR") -TMP_ROOT="$TMPDIR/root" - -zfs create \ - -o canmount=off \ - -o mountpoint=none \ - "$RPOOL/ROOT" || - die "failed to create linux dataset $RPOOL/ROOT" - -zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o "com.delphix:initial-version=$DLPX_VERSION" \ - -o "com.delphix:current-version=$DLPX_VERSION" \ - "$RPOOL/ROOT/$FSNAME" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME" - -zfs create \ - -o canmount=noauto \ - -o mountpoint="$TMP_ROOT" \ - "$RPOOL/ROOT/$FSNAME/root" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/root" - -zfs mount "$RPOOL/ROOT/$FSNAME/root" || - die "failed to mount $RPOOL/ROOT/$FSNAME/root in temporary dir $TMP_ROOT" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/home" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/home" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/data" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/data" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/log" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/log" - -mkdir -p "$TMP_ROOT/export/home" -mount -F zfs "$RPOOL/ROOT/$FSNAME/home" "$TMP_ROOT/export/home" - -mkdir -p "$TMP_ROOT/var/delphix" -mount -F zfs "$RPOOL/ROOT/$FSNAME/data" "$TMP_ROOT/var/delphix" - -mkdir -p "$TMP_ROOT/var/log" -mount -F zfs "$RPOOL/ROOT/$FSNAME/log" "$TMP_ROOT/var/log" - -( - cd "$TMP_ROOT" - cpio -imu -) <"$ARCHIVE_DIR/os-root.cpio" 2>&1 || - die "failed to unpack os-root.cpio" - -# -# /export/home and /var/delphix datasets are not mounted automatically. -# On Illumos this kind of mounting logic was performed by dxinit and svc-boot. -# On Linux /etc/fstab is used instead. These mounts need to happen before -# the zfs-import service runs. -# -cat <<-EOF >"$TMP_ROOT/etc/fstab" || die "Failed to setup /etc/fstab" - $RPOOL/ROOT/$FSNAME/home /export/home zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/ROOT/$FSNAME/data /var/delphix zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/ROOT/$FSNAME/log /var/log zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/crashdump /var/crash zfs defaults,x-systemd.before=zfs-import-cache.service,x-systemd.before=kdump-tools.service 0 0 -EOF - -# -# On some platforms (currently AWS and new enough versions of ESX), the -# engine's UUID as reported by the os-level tools changes after -# migration. This is because of a confusion over the proper endianness of the -# first three fields of the UUID. To avoid having engine UUIDs change after -# migration, we store the current UUID before we finish the migration, and -# then use that on Linux. -# -/opt/delphix/server/bin/bos_mgmt get_system_uuid appliance >"$TMP_ROOT/etc/engine-uuid" || - die "failed to copy engine uuid" - -# -# Set things up for the FreeBSD bootloader to provide an option for -# booting into the Linux environment This happens in two steps: -# [1] We place vmlinuz and initrd from our migration image into -# the /boot directory of the current system, so the bootloader -# can load them. -# [2] Add a new option in the bootloader's menu for Linux. This is -# done by appending the configuration of the new option in our -# current menu file. -# -# Note: We reverse sort below based on a vmlinuz & initrd's name -# to get the latest version of them. In reality, there is always -# only one version but in the case that there is an issue in -# how we construct the images we always want to get the latest. -# -VMLINUZ=$(find "$TMP_ROOT/boot/vmlinuz-"* | sort -r | head -n 1) -INITRD_IMG=$(find "$TMP_ROOT/boot/initrd.img-"* | sort -r | head -n 1) -cp "$VMLINUZ" /boot || - die "failed to copy $VMLINUZ to /boot" -cp "$INITRD_IMG" /boot || - die "faield to copy $INITRD_IMG to /boot" - -# -# This command loads the Linux Kernel Compressed Executable in memory -# from the OK prompt of the FreeBSD bootloader. Note that we pass it the -# lxcmdline variable, which will be parsed by the bootloader. The reason -# why we create a separate variable for the cmdline arguments instead of -# just adding them in-line is because otherwise the command would be -# greater than 256 characters and fail to be parsed by the bootloader. -# We have to be careful to escape ${lxcmdline} properly so that it is -# not resolved by the current script. -# -LOAD_VMLINUZ_OK_CMD="load /boot/$(basename "$VMLINUZ") \${lxcmdline}" - -# -# Essentially we want the command line to be as close as possible to -# what is set by our grub configuration for fresh Linux installs. Our -# customizations to the grub command line can be found in the -# delphix-platform project at etc/default/grub.d/override.cfg. -# Here is an overview: -# -# - We specify the root filesystem type to be ZFS and its root dataset -# the one that we just created for the migration image. -# -# - We carry-over the following settings from delphix-platform: -# console, ipv6 and crashkernel. -# -# - We specify zfsforce=1 because otherwise we would be failing at -# pool import because of ZFS's hostid check. This argument is added -# specifically for the migration case and doesn't appear on a fresh -# Linux install. -# -# Note that we use an array of strings instead of one whole string -# that we break to multiple lines in order to divide the arguments -# in logical pieces while ensuring that their quotes are escaped -# correctly and in accordance with our linters/checkstyles. This -# should be taken into account for all cases of string arrays for -# the rest of this file that have to do with adding commands in the -# bootloader configuration. -# -LX_CMDLINE=( - "root=ZFS=$RPOOL/ROOT/$FSNAME/root" - 'console=tty0 console=ttyS0,115200n8' - 'ipv6.disable=1' - 'crashkernel=256M,high' - 'crashkernel=256M,low' - 'zfsforce=1' - 'mitigations=off' - 'elevator=noop' - 'init_on_alloc=0' - 'usbcore.nousb=1' -) - -# -# This command loads the initial RAM disk as the initial root filesystem -# which is needed as part of the booting process. -# -LOAD_INITRD_OK_CMD="load -t rootfs /boot/$(basename "$INITRD_IMG")" - -# -# This is the command to be issued whenever our new option in the -# bootloader menu is chosen. It is FICL code (a Forth-inspired -# command language) that is basically read as: -# arg func; arg func; func. -# In our case we: -# [1] Evaluate the command to load vmlinuz -# [2] Evaluate the command to load initrd -# [3] boot (with the above two loaded) -# -MAIN_MENU_FICL=( - "s\\\" $LOAD_VMLINUZ_OK_CMD\\\" evaluate" - "s\\\" $LOAD_INITRD_OK_CMD\\\" evaluate" - boot -) - -# -# Note: We always have 7 options in our bootloader at any given time -# in the latest 5.3 version, thus we use the index 8 for the new -# option below. -# -{ - echo 'set lxcmdline="'"${LX_CMDLINE[*]}"'"' - # - # Add non-ansi and ansi version for the name of the option. - # - echo 'set mainmenu_caption[8]="Boot Delphix on [L]inux"' - echo 'set mainansi_caption[8]="Boot Delphix on ^[1mL^[minux version"' - - # - # 76 is ascii for capital 'L', see caption above - # - echo 'set mainmenu_keycode[8]=76' - - # - # If someone selects our new option, boot into Linux. - # - echo 'set mainmenu_command[8]="'"${MAIN_MENU_FICL[*]}"'"' -} >>/boot/menu.rc.local || - die "failed to update /boot/menu.rc.local" - -# -# Note that the new /var/delphix should not contain any useful data as it will -# be replaced by a clone of the current /var/delphix dataset in dx_execute. -# We do not clone the current /var/delphix right away as the clone will be -# outdated by the time dx_execute is called. Before deleting the dataset we -# first verify that it doesn't contain anything useful. -# -( - cd "$TMP_ROOT/var/delphix" || die "failed to cd into $TMP_ROOT/var/delphix" - # /var/delphix should be empty - [[ $(find . -mindepth 1 | wc -l) -eq 0 ]] || - die "linux dataset for /var/delphix contains unexpected files" -) || die "verification of /var/delphix failed" - -__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING" - -umount "$TMP_ROOT/var/log" || - die "couldn't unmount linux dataset $TMP_ROOT/var/log" -zfs destroy "$RPOOL/ROOT/$FSNAME/data" || - die "failed to destroy linux dataset $RPOOL/ROOT/$FSNAME/data" -umount "$TMP_ROOT/export/home" || - die "couldn't unmount linux dataset $TMP_ROOT/export/home" - -zfs umount "$RPOOL/ROOT/$FSNAME/root" || - die "couldn't unmount linux dataset $RPOOL/ROOT/$FSNAME/root" -zfs set mountpoint=/ "$RPOOL/ROOT/$FSNAME/root" || - die "could not set mountpoint for linux dataset $RPOOL/ROOT/$FSNAME/root" - -rm -rf "$TMPDIR" || - die "failed to destroy temporary directory $TMPDIR" - -report_progress_inc 100 - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_delete b/live-build/misc/migration-scripts/dx_delete deleted file mode 100755 index f90a764e..00000000 --- a/live-build/misc/migration-scripts/dx_delete +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -set -o pipefail - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -# -# Here, we assume Linux dataset names starts with "rpool/ROOT", and also -# assume that there can be at most one migration dataset applied in current OS -# since migration is only supported from *fixed* 5.3.X version to 6.0.X version. -# Also, it is not possible to get a version from dataset name in Linux since -# names are randomly generated. -# - -LX_DATASET="rpool/ROOT" -if ! zfs list ${LX_DATASET} &>/dev/null; then - echo "Linux dataset '${LX_DATASET}' is not installed." - exit 0 -fi - -LX_RDS_PARENT=$(zfs list -o name -H -d 1 "$LX_DATASET" | tail -n 1) -[[ -n $LX_RDS_PARENT ]] || die "could not find Linux RDS parent dataset" -LX_CONTAINER="${LX_RDS_PARENT##*/}" -LX_RDS_TMP_ROOT_MOUNT="/tmp/$LX_CONTAINER/root" - -MDS_SNAPNAME="MDS-CLONE-upgradeverify" -MDS_CLONE=domain0/$MDS_SNAPNAME -SVC=svc:/system/delphix/postgres:$MDS_SNAPNAME -PG_DATA=/$MDS_CLONE/db - -BOOT_MENU=/boot/menu.rc.local -BOOT_MENU_COPY=$BOOT_MENU.copy - -function dx_apply_cleanup() { - # - # Restore original version of the bootloader - # - if [[ -e $BOOT_MENU_COPY ]]; then - mv $BOOT_MENU_COPY $BOOT_MENU || - die "failed to restore copy of $BOOT_MENU" - fi - - rm -f /boot/vmlinuz-* /boot/initrd.img-* || - die "failed to destroy previously copied Linux kernel data" -} - -function dx_verify_cleanup() { - # - # dx_verify creates a clone of the MDS dataset and the postgres - # service as well as a clone of /var/delphix for masking validation. - # If any are left, clean them up. (This logic is based on dx_manage_pg - # stop and cleanup.) - # - local sta - sta=$(svcs -Ho sta $SVC) - if [[ -n "$sta" ]]; then - /usr/sbin/svcadm disable -s $SVC || - die "unable to disable SMF service: $SVC" - /usr/sbin/svccfg delete -f $SVC || - die "unable to delete SMF service: $SVC" - fi - - # - # Clean up the postmaster.pid leftover from the main MDS in the - # snapshot. This needs to be done after disabling the SMF service in - # case the postmaster.pid was instead from the active clone. - # - if [[ -e $PG_DATA/postmaster.pid ]]; then - echo "renaming $PG_DATA/postmaster.pid to $PG_DATA/postmaster.pid.original" - mv $PG_DATA/postmaster.pid $PG_DATA/postmaster.pid.original || - echo "failed to move postmaster.pid file" - fi - - [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && - zfs destroy -R "domain0/mds@$MDS_SNAPNAME" - [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && - die "unable to cleanup domain0/mds@$MDS_SNAPNAME" - - runningVar=$(mount | awk '/^\/var\/delphix /{ print $3 }') - [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && - zfs destroy -R "$runningVar@$MDS_SNAPNAME" - [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && - die "unable to cleanup $runningVar@$MDS_SNAPNAME" -} - -dx_apply_cleanup -dx_verify_cleanup - -# -# Recursively destroy and force-unmount the Linux Root Dataset. Even though all the -# datasets are cleaned up in a normal execution of Verify, this handles the case when -# it was cancelled early. -# -zfs destroy -rf ${LX_DATASET} || die "Failed to destroy Linux dataset '${LX_DATASET}'" -rm -rf "$LX_RDS_TMP_ROOT_MOUNT" diff --git a/live-build/misc/migration-scripts/dx_execute b/live-build/misc/migration-scripts/dx_execute deleted file mode 100755 index fdef0ca6..00000000 --- a/live-build/misc/migration-scripts/dx_execute +++ /dev/null @@ -1,237 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Finalizes the Linux image in preparation for the reboot now that the app -# stack is quiesced; sets up the FreeBSD bootloader to boot into that Linux -# image next time we reboot; finally reboots the system. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -set -o pipefail - -DX_UPG_PAUSE="${BASH_SOURCE%/*}/dx_upg_pause_options" -DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -function usage() { - [[ -n "$1" ]] && echo "$1" - cat </dev/null; then - zfs destroy -R "$dataset" || - die "Failed to destroy leftover dataset '$dataset'" - fi -} - -opt_s=false -while getopts :hs c; do - case "$c" in - \?) usage "Invalid option: -$OPTARG." ;; - h) usage ;; - s) eval "opt_$c=true" ;; - esac -done - -[[ "$(uname -s)" == "SunOS" ]] || die "script can only be used in illumos" - -# -# Undo any existing default timeout commands in the bootloader's menu. -# -sed -i '/menu_timeout_command/d' /boot/menu.rc.local -[[ "$(grep -cF 'menu_timeout_command' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup previous default command" - -# -# Get the RDS and the current ZFS pool that we're currently using. -# -RDS=$(mount | awk '/^\/ /{ print $3 }') -RPOOL=${RDS%%/*} - -CURRENT_DDS=$(dirname "$(mount | awk '/^\/opt\/delphix /{ print $3 }')") -CURRENT_VERSION=$(basename "$CURRENT_DDS") - -# -# Ensure that the expected Linux dataset layout exists. -# Note that this excludes the "data" dataset which was destroyed in dx_apply. -# -[[ $(zfs list -o name -Hr "$RPOOL/ROOT" | wc -l) -eq 5 ]] || - die "could not find the expected linux dataset layout" -LX_RDS_PARENT=$(zfs list -o name -H -d 1 "$RPOOL/ROOT" | tail -n 1) -[[ -n $LX_RDS_PARENT ]] || die "could not find Linux RDS parent dataset" -LX_RDS="$LX_RDS_PARENT/root" - -# -# Ensure that the expected bootloader fields are there. -# -[[ "$(grep -cF 'set lxcmdline' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no lxcmdline variable in the bootloader menu" -[[ "$(grep -cF 'mainmenu_caption[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no caption for the Linux option in the bootloader menu" -[[ "$(grep -cF 'mainmenu_caption[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no keycode for the Linux option in the bootloader menu" -[[ "$(grep -cF 'mainmenu_command[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no command for the Linux option in the bootloader menu" -[[ "$(grep -c "$LX_RDS" /boot/menu.rc.local)" -eq 1 ]] || - die "the expected Linux RDS ($LX_RDS) was either not found or has" \ - "been specified more than once in the bootloader's menu file" - -# -# Re-mount the root dataset and run the migration logic. -# -rm -rf /var/delphix/migration -mkdir -p /var/delphix/migration || - die "failed to create /var/delphix/migration" -MIGRATION_SCRIPT="/opt/delphix/migration/migrate_config.py" -LX_CONTAINER="${LX_RDS_PARENT##*/}" -LX_RDS_MOUNT="/tmp/delphix.$LX_CONTAINER/root" -mkdir -p "$LX_RDS_MOUNT" -mount -F zfs -o ignoremountpoint "$LX_RDS" "$LX_RDS_MOUNT" || - die "failed to mount the root Linux dataset" -"${LX_RDS_MOUNT}${MIGRATION_SCRIPT}" pre-upgrade \ - >>/var/delphix/migration/log 2>&1 || die "failed to run migration" - -__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION" - -# -# Create a flag file that notifies the delphix-migration service that -# post-reboot migration logic should be run. Note that the /var/delphix -# dataset is carried over into Linux. -# -touch /var/delphix/migration/perform-migration || - die "failed to create delphix-migration flag file" - -# -# Let the app-stack know that an "OS migration" upgrade is being performed. -# -cat <<-EOF >"/var/dlpx-update/upgrade.properties" || - UPGRADE_TYPE=OS_MIGRATION - UPGRADE_BASE_VERSION=$CURRENT_VERSION -EOF - die "failed to create upgrade.properties" - -# -# Create linux /var/delphix dataset from a clone of the current -# /var/delphix dataset. We want to do this last as this will carry-over all -# the logs and the /var/delphix/migration directory into Linux. -# -LX_VAR_DLPX="$LX_RDS_PARENT/data" -CUR_VAR_DLPX=$(zfs list -Ho name /var/delphix) -[[ -n $CUR_VAR_DLPX ]] || die "could not determine current /var/delphix dataset" -cleanup_leftover_dataset "${CUR_VAR_DLPX}@migration" -zfs snapshot "${CUR_VAR_DLPX}@migration" || - die "failed to create snapshot '${CUR_VAR_DLPX}@migration'" -zfs clone \ - -o mountpoint=legacy \ - "${CUR_VAR_DLPX}@migration" \ - "$LX_VAR_DLPX" || - die "failed to clone dataset ${CUR_VAR_DLPX}@migration" - -# -# Create a clone of the current /export/home dataset. It will be kept as -# a backup in case Delphix support or services have left useful files in -# there since all rpool/versions datasets will be cleaned-up on next upgrade. -# -CUR_HOME=$(zfs list -Ho name /export/home) -[[ -n $CUR_HOME ]] || die "could not determine current /export/home dataset" -zfs list "$CUR_HOME" &>/dev/null || - die "'$CUR_HOME' is not a valid zfs dataset" -cleanup_leftover_dataset "${CUR_HOME}@migration" -cleanup_leftover_dataset rpool/illumos-home -zfs snapshot "${CUR_HOME}@migration" || - die "failed to create snapshot '${CUR_HOME}@migration'" -zfs clone \ - -o mountpoint=legacy \ - "${CUR_HOME}@migration" \ - rpool/illumos-home || - die "failed to clone dataset ${CUR_HOME}@migration" - -# -# Create a snapshot of /mds to make it easier to rollback to it if needed. -# -cleanup_leftover_dataset domain0/mds@migration -zfs snapshot domain0/mds@migration || - die "failed to create snapshot domain0/mds@migration" - -# -# Increase the boot delay to 20 seconds to ease bootloader access during -# the migration for debugging purposes. -# -sed -i '/autoboot_delay/c\set autoboot_delay=20' /boot/menu.rc.local || - die "Failed to set boot delay in bootloader" - -# -# Read the command from option 8 which should be the one that boots -# into Linux. Then make it so that the same command runs whenever we -# hit the timer of the FreeBSD bootloader menu. -# -MAIN_MENU_LINUX_CMD=$(grep -F 'mainmenu_command[8]' /boot/menu.rc.local | - cut -d = -f 2-) -echo "set menu_timeout_command=$MAIN_MENU_LINUX_CMD" >>/boot/menu.rc.local - -# -# Notify the UI that the system is rebooting. -# This could hang if the JVM is in a bad state, so we use a timeout. -# -timeout 10 /opt/delphix/server/bin/jmxtool boot upgrade server - -$DX_UPG_PAUSE --pause "PAUSE_IN_DX_EXECUTE_BEFORE_RESTART" || - die "failed to pause fully on stress option" - -__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT" - -# Constants used by the uadmin syscall. -A_SHUTDOWN=2 -AD_POWEROFF=6 -AD_BOOT=1 - -# reboot or shutdown -if $opt_s; then - uadmin $A_SHUTDOWN $AD_POWEROFF -else - uadmin $A_SHUTDOWN $AD_BOOT -fi - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_prepare b/live-build/misc/migration-scripts/dx_prepare deleted file mode 100755 index 6430016e..00000000 --- a/live-build/misc/migration-scripts/dx_prepare +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is a no-op and only exist to satisfy dx_unpack.sh -# from illumos. -# - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_upg_pause_options b/live-build/misc/migration-scripts/dx_upg_pause_options deleted file mode 100755 index 6b9e0e79..00000000 --- a/live-build/misc/migration-scripts/dx_upg_pause_options +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017, 2019 by Delphix. All rights reserved. -# - -# -# Utility for managing upgrade pause options. -# Supported pause options listed below. -# -# Note that those pause options are intended to be run exclusively for -# migration testing, before the upgrade reboot. -# - -PAUSE_OPTIONS_JSON=$( - cat <<-EOF - { - "PAUSE_IN_DX_EXECUTE_BEFORE_RESTART": { - "location": "right before reboot" - }, - "PAUSE_IN_DXSVCINIT_BEFORE_UPGRADE": { - "location": "post-reboot before any app upgrade logic" - }, - "PAUSE_BEFORE_UPGRADE_COMMIT": { - "location": "mgmt service (before point of no return)" - }, - "PAUSE_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (before point of no return)" - }, - "PAUSE_IN_SVC_ROLLBACK_BEFORE_ROLLBACK": { - "location": "rollback service" - }, - "PAUSE_BEFORE_REWIND_CHECKPOINT": { - "location": "rollback service" - }, - "PAUSE_AFTER_REWIND_CHECKPOINT": { - "location": "rollback service" - }, - "PAUSE_IN_DXSVCINIT_AFTER_ROLLBACK": { - "location": "post-rollback before app stack starts" - }, - "PAUSE_IN_SVC_MGMT_BEFORE_STACK_STARTUP": { - "location": "before app stack starts" - } - } - EOF -) - -PAUSE_DATASET=rpool/pause_options -DLPX_ENTERED=com.delphix:entered - -function die() { - echo "$(basename "$0"): $*" 1>&2 - exit 1 -} - -function usage() { - echo "$(basename "$0"): $*" >&2 - echo "Usage:" - echo " $(basename "$0") --get-options-json" - echo " Prints out a detailed json of all the supported stress options." - echo " $(basename "$0") --set " - echo " Sets the specified pause option. Non-zero exit code on failure." - echo " $(basename "$0") --unset " - echo " Unsets the specified pause option. Non-zero exit code on failure." - echo " $(basename "$0") --get " - echo " Returns exit code 3 if given pause option is set, 0 if unset." - echo " Any other non-zero exit code is an unexpected error." - echo " $(basename "$0") --pause " - echo " Does not terminate until the pause option is unset." - echo " Non-zero exit code if errors hit, 0 if wait successful." - echo " $(basename "$0") --wait-for-pause " - echo " Does not terminate until the pause option is hit." - echo " Non-zero exit code if errors hit, 0 if wait successful." - exit 2 -} - -function check_valid_option() { - local arg="$1" - - local get_names - get_names=$( - cat <<-EOF - import json, sys - obj=json.load(sys.stdin) - print(" ".join(obj.keys())) - EOF - ) - local options - options=$(echo "$PAUSE_OPTIONS_JSON" | python -c "$get_names") - - for option in $options; do - [[ "$option" == "$arg" ]] && return - done - usage "Unsupported pause option '$arg'" -} - -function set_pause_option() { - local option="$1" - local entered="$2" - - if ! zfs list "$PAUSE_DATASET" &>/dev/null; then - zfs create -o mountpoint=legacy "$PAUSE_DATASET" || return 1 - fi - - if ! zfs list "$PAUSE_DATASET/$option" &>/dev/null; then - zfs create -o mountpoint=legacy -o "$DLPX_ENTERED=$entered" \ - "$PAUSE_DATASET/$option" || return 1 - else - zfs set "$DLPX_ENTERED=$entered" "$PAUSE_DATASET/$option" || return 1 - fi - return 0 -} - -# Requires option to be set, otherwise returns non-zero code. -function enter_pause_option() { - local option="$1" - zfs set "$DLPX_ENTERED=$entered" "$PAUSE_DATASET/$option" || return 1 - return 0 -} - -function unset_pause_option() { - local option="$1" - - if zfs list "$PAUSE_DATASET/$option" &>/dev/null; then - zfs destroy "$PAUSE_DATASET/$option" || return 1 - fi - return 0 -} - -function get_pause_option() { - local option="$1" - zfs list "$PAUSE_DATASET/$option" &>/dev/null && return 3 - return 0 -} - -# Verify arguments -[[ $# -ge 1 ]] || usage "missing arguments" - -case "$1" in -"--set") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - set_pause_option "$2" "false" - exit $? - ;; -"--unset") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - unset_pause_option "$2" - exit $? - ;; -"--get") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - get_pause_option "$2" - exit $? - ;; -"--pause") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - - # If option not set, nothing to do. - get_pause_option "$2" && exit 0 - - echo -e "Pausing on $2 ... \\c" - - # Mark as entered - enter_pause_option "$2" || die "failed to mark entered" - - get_pause_option "$2" - enabled=$([[ $? -eq 3 ]] && echo true || echo false) - while $enabled; do - sleep 1 - - # - # Enjoy this spinner while you wait... - # (copied this genius piece of code from start_mgmt_server_jvm) - # - case $((SECONDS % 4)) in - 0) echo -e "\\b\\b- \\c" ;; - 1) echo -e "\\b\\b\\ \\c" ;; - 2) echo -e "\\b\\b| \\c" ;; - 3) echo -e "\\b\\b/ \\c" ;; - esac - - get_pause_option "$2" - enabled=$([[ $? -eq 3 ]] && echo true || echo false) - done - echo "\\b\\b\\b done." - ;; -"--wait-for-pause") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - - # Check if the option was set - get_pause_option "$2" && die "pause option $2 not set" - - echo -e "Waiting for pause on $2 ... \\c" - entered=false - while ! $entered; do - sleep 1 - entered=$(zfs get -Ho value $DLPX_ENTERED "$PAUSE_DATASET/$2") - # shellcheck disable=SC2181 - [[ $? -eq 0 ]] || die "failed to get property for $2" - done - echo "done." - ;; -"--get-options-json") - echo "$PAUSE_OPTIONS_JSON" - exit $? - ;; -*) - usage "illegal option '$1'" - ;; -esac - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_upg_stress_options b/live-build/misc/migration-scripts/dx_upg_stress_options deleted file mode 100755 index addd500f..00000000 --- a/live-build/misc/migration-scripts/dx_upg_stress_options +++ /dev/null @@ -1,480 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2016, 2019 by Delphix. All rights reserved. -# - -# -# Utility for getting and setting upgrade stress options. -# Supported failure points listed below. -# -# This file may be sourced so the variable/function names are made as unique -# as possible in addition to the double underscores in front. -# -# Note that those stress options are intended to be run exclusively for -# migration testing, before the upgrade reboot. -# - -__STRESS_OPTIONS_JSON=$( - cat <<-EOF - { - "STRESS_DX_APPLY_FAIL_AFTER_VERSION_CHECK": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after version check.", - "auto_unset": true - }, - "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before dx_apply dataset cleanup.", - "auto_unset": true - }, - "STRESS_DX_INSTALL_ARCHIVE_FAIL_AFTER_VERIFY_DX_ARCHIVE": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after verify_dx_archive.", - "auto_unset": true - }, - "STRESS_DX_VERIFY_FAIL_HOTFIX": { - "location": "pre-reboot", - "err_msg": "Stress option triggered to fail hotfix check.", - "auto_unset": true - }, - "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after upgrade-verify.jar.", - "auto_unset": true - }, - "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after migrating OS config.", - "auto_unset": true - }, - "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before reboot.", - "auto_unset": true - }, - "STRESS_APPLY_JOB_CLEANUP_BEFORE_DISABLE_UPGRADE_MODE": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before cleaning up upgrade mode.", - "auto_unset": true - }, - "STRESS_DXSVCINIT_FAIL_AFTER_UPGRADE_MODE": { - "location": "boot service", - "err_msg": "Stress option triggered after upgrade mode.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_MASKING_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before upgrading masking.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_MASKING_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered after upgrading masking.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_UPGRADE_COMMIT": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before upgrade commit.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_DOMAIN_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before domain upgrade.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before checkpoint discard.", - "auto_unset": true - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered before do_rollback.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_DX_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered before dx_rollback call.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_AFTER_DX_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered after dx_rollback call.", - "auto_unset": true - }, - "STRESS_ROLLBACK_SVC_FAIL_AFTER_REMOUNT_OPT": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered after remounting /opt/delphix.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_CLEARING_DEPS": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered before clearing dependent services.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_REENABLING_BOOT": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered before re-enabling boot service.", - "auto_unset": false - }, - "STRESS_CKPT_UTIL_FAIL_AFTER_EXPORT_DOMAIN": { - "location": "rollback service (OS)", - "err_msg": "Stress option triggered after exporting domain0.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_COMMIT_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered between upgrade commit and checkpoint discard", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_BOOTLOADER_REWRITE": { - "location": "mgmt service (after point of no return, migration only)", - "err_msg": "Stress option triggered before rewriting bootloader.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_UPGRADE_COMMIT": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered after upgrade commit.", - "auto_unset": true - }, - "STRESS_UPGRADEMANAGER_FAIL_IN_START": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered in upgrade manager.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_POST_MIGRATION_CLEANUP": { - "location": "mgmt service (after point of no return, migation only)", - "err_msg": "Stress option triggered before performing post-migration cleanup.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_MIGRATION_MARKED_COMPLETED": { - "location": "mgmt service (after point of no return, migation only)", - "err_msg": "Stress option triggered before marking migration as completed.", - "auto_unset": true - }, - "STRESS_POSTCLEANUP_FAIL_BEFORE_ENABLE_SOURCES": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered before enabling sources.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEGIN_MIGRATION_SERVICE": { - "location": "migration service", - "err_msg": "Stress option triggered starting migration service.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEFORE_DOMAIN0_IMPORT": { - "location": "migration service", - "err_msg": "Stress option triggered before importing domain0.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_DOMAIN0_IMPORT": { - "location": "migration service", - "err_msg": "Stress option triggered after importing domain0.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_ZFS_MOUNT": { - "location": "migration service", - "err_msg": "Stress option triggered after zfs mount -a.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_MOUNT_PERMISSION_FIX": { - "location": "migration service", - "err_msg": "Stress option triggered after fixing mount permissions.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEFORE_MIGRATE_CONFIG": { - "location": "migration service", - "err_msg": "Stress option triggered before calling migrate-config.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_MIGRATE_CONFIG": { - "location": "migration service", - "err_msg": "Stress option triggered after calling migrate-config.", - "auto_unset": true - } - } - EOF -) - -__STRESS_DATASET=rpool/stress_options -__DLPX_PANIC=com.delphix:panic -__DLPX_SVC_MGMT=svc:/system/delphix/mgmt - -function __dx_upg_stress_options_usage() { - cat <<-EOF >&2 - $(basename "$0"): $* - Usage: - $(basename "$0") --get-options-json - Prints out a detailed json of all the supported stress options. - $(basename "$0") --set - Set the specified stress option to die on trigger. - Non-zero exit status on failure. - $(basename "$0") --set-panic - Set the specified stress option to system panic on trigger. - Non-zero exit status on failure. - $(basename "$0") --unset - Unset the specified stress option. - Non-zero exit status on failure. - $(basename "$0") --get - Returns exit code 3 if given stress option is set to die, - 0 if not. Any other non-zero exit code is an unexpected error. - $(basename "$0") --get-panic - Returns exit code 3 if given stress option is set to panic, - 0 if not. Any other non-zero exit code is an unexpected error. - $(basename "$0") --panic-or-get-unset - If the stress option is set to panic, unset and panic. - Otherwise returns exit code: - 0 - the stress option is unset. - 3 - the stress option is set, and successfully unset it. - 4 - the stress option is set, but unsuccessfully unset it. - Any other non-zero exit code is an unexpected error. - EOF - exit 2 -} - -function __check_valid_option() { - local arg="$1" - - local get_names - get_names=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(" ".join(obj.keys())) - EOF - ) - local options - options=$(python -c "$get_names") - - for option in $options; do - [[ "$option" == "$arg" ]] && return - done - __dx_upg_stress_options_usage "Unsupported stress option '$arg'" -} - -function __set_stress_option() { - local option="$1" - local panic="$2" - - if ! zfs list "$__STRESS_DATASET" &>/dev/null; then - zfs create -o mountpoint=legacy "$__STRESS_DATASET" || return 1 - fi - - if ! zfs list "$__STRESS_DATASET/$option" &>/dev/null; then - zfs create -o mountpoint=legacy -o "$__DLPX_PANIC=$panic" \ - "$__STRESS_DATASET/$option" || return 1 - else - zfs set "$__DLPX_PANIC=$panic" "$__STRESS_DATASET/$option" || return 1 - fi - return 0 -} - -function __unset_stress_option() { - local option="$1" - - if zfs list "$__STRESS_DATASET/$option" &>/dev/null; then - zfs destroy "$__STRESS_DATASET/$option" || return 1 - fi - return 0 -} - -function __get_stress_option() { - local option="$1" - local panic="$2" - - zfs list "$__STRESS_DATASET/$option" &>/dev/null && - [[ $(zfs get -Ho value "$__DLPX_PANIC" "$__STRESS_DATASET/$option") == "$panic" ]] && return 3 - return 0 -} - -function __get_unset_stress_option() { - local option="$1" - local panic="$2" - - __get_stress_option "$option" "$panic" - if [[ $? -eq 3 ]]; then - __unset_stress_option "$option" || return 4 - return 3 - fi - return 0 -} - -function __handle_panic_stress_option() { - local option="$1" - local unset="$2" - local get_func="__get_stress_option" - $unset && get_func="__get_unset_stress_option" - - $get_func "$option" "true" - local stress_ret - stress_ret=$? - - if [[ $stress_ret -eq 3 ]]; then - uadmin 2 1 - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - echo "'$get_func' returned $stress_ret" - fi -} - -# -# Fails if the given stress option was set. -# Also fails if we are in debug mode and hit an unexpected return code. -# -# This function should only be used when sourcing this file. The script -# sourcing this file must implement the function "die" or "stress_die" for -# this method to fail correctly. -# -# A caller may define a "stress_die" function if they want extra functionality -# besides the original "die" method. See "dx_execute" for an example of this. -# If no "stress_die" is defined, it will fall back to using the original "die" -# method for failing. -# -function __trigger_stress_option() { - local option=$1 - local delphix_debug - delphix_debug=$(svcprop -p delphix/debug $__DLPX_SVC_MGMT) - - __handle_panic_stress_option "$option" "false" - - local get_err_msg - get_err_msg=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(obj["$option"]["err_msg"]) - EOF - ) - local message - message=$(python -c "$get_err_msg") - - __get_stress_option "$option" "false" - local stress_ret=$? - if [[ $stress_ret -eq 3 ]]; then - type stress_die >/dev/null 2>&1 && stress_die "$message" - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "$message" - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "'__get_stress_option' returned $stress_ret" - fi -} - -# -# Fails if the given stress option was set (unsets it too). -# Also fails if we are in debug mode and hit an unexpected return code. -# -# This function should only be used when sourcing this file. The script -# sourcing this file must implement the function "die" or "stress_die" for -# this method to fail correctly. -# -# A caller may define a "stress_die" function if they want extra functionality -# besides the original "die" method. See "dx_execute" for an example of this. -# If no "stress_die" is defined, it will fall back to using the original "die" -# method for failing. -# -function __trigger_unset_stress_option() { - local option="$1" - local delphix_debug - delphix_debug=$(svcprop -p delphix/debug $__DLPX_SVC_MGMT) - - __handle_panic_stress_option "$option" "true" - - local get_err_msg - get_err_msg=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(obj["$option"]["err_msg"]) - EOF - ) - local message - message=$(python -c "$get_err_msg") - - __get_unset_stress_option "$option" "false" - local stress_ret=$? - if [[ $stress_ret -eq 3 ]]; then - type stress_die >/dev/null 2>&1 && stress_die "$message" - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "$message" - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "'__get_unset_stress_option' returned $stress_ret" - fi -} - -[[ $# -ge 1 ]] || __dx_upg_stress_options_usage "missing arguments" -case "$1" in -"--set") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __set_stress_option "$2" "false" - exit $? - ;; -"--set-panic") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __set_stress_option "$2" "true" - exit $? - ;; -"--unset") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __unset_stress_option "$2" "false" - exit $? - ;; -"--get") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __get_stress_option "$2" "false" - exit $? - ;; -"--get-panic") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __get_stress_option "$2" "true" - exit $? - ;; -"--panic-or-get-unset") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - - # Called when the stress option is set and successfully unset, exit 3. - function stress_die() { - echo "$*" - exit 3 - } - - # Called when the stress option is set but failed to unset, exit 4. - function die() { - echo "$*" - exit 4 - } - - __trigger_unset_stress_option "$2" - exit 0 - ;; -"--get-options-json") - echo "$__STRESS_OPTIONS_JSON" - exit $? - ;; -"--source") - # do nothing - ;; -*) - __dx_upg_stress_options_usage "illegal option '$1'" - ;; -esac - -# Do not exit here because sourcing this file will exit the parent. diff --git a/live-build/misc/migration-scripts/dx_verify b/live-build/misc/migration-scripts/dx_verify deleted file mode 100755 index 84b9368b..00000000 --- a/live-build/misc/migration-scripts/dx_verify +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default. -# -umask 0022 - -export MDS_SNAPNAME="MDS-CLONE-upgradeverify" - -DEBUG=false - -MDS_CLONE=domain0/$MDS_SNAPNAME -UPGRADE_VERIFY_PATH=/opt/delphix/server/lib/exec/upgrade-verify -UPGRADE_VERIFY_JAR=$UPGRADE_VERIFY_PATH/upgrade-verify.jar -LOG_DIR=/var/delphix/server/upgrade-verify -MGMT_FMRI=svc:/system/delphix/mgmt:default -BINDIR=/opt/delphix/server/bin -DX_MANAGE_PG=$BINDIR/dx_manage_pg - -function usage() { - echo "usage: $(basename "$0") -v -o -f -l " - echo " -v : The Delphix version number we're upgrading to" - echo " -o : The output file verification should generate containing an upgrade verification report" - echo " -f : The format of the upgrade verification report to generate" - echo " -l : The locale the upgrade verification report should target" - echo " -d: Disable Consistent MDS ZFS Data Utility feature" - exit 1 -} - -function cleanup() { - if ! $DEBUG; then - if [[ -n "$root" ]]; then - cleanup_postgres - cleanup_datasets - cleanup_masking - fi - fi -} - -function die() { - report "$(basename "$0"): $*" >&2 - - # - # If we have MDS postgres logs, save them before cleaning up so - # they can be analyzed later. - # - if [[ -f "/$MDS_CLONE/db/stdout.log" ]]; then - ( - cd "/$MDS_CLONE/db" || exit 1 - tar -cvf /var/delphix/server/db/pg_log.upgrade.$$.tar \ - stdout.log pg_log - ) - fi - - cleanup - exit 1 -} - -function report() { - echo "$(date +%T:%N:%z): $1" -} - -function report_progress_inc() { - echo "Progress increment: $(date +%T:%N%z), $1, $2" -} - -function mount_datasets() { - local version=$1 - local rds - - rds=$(zfs list -o name -H -d3 rpool/ROOT | grep "delphix.*/root") - [[ -n $rds ]] || die "could not find migration dataset for version $version" - - root=$(mktemp -d) || die "unable to create temporary directory" - chmod 755 "$root" || die "unable to set permissions for $root" - mount -F zfs -o ignoremountpoint "$rds" "$root" || die "unable to mount $rds" - - #TODO dummy file for UpgradeVerify see LX-1808 - touch /var/dlpx-update/"$version"/dx_upg_stress_options - - # - # We need a clone of /var/delphix for masking migration scripts. - # - runningVar=$(mount | awk '/^\/var\/delphix /{ print $3 }') - zfs snapshot "$runningVar@$MDS_SNAPNAME" || - die "unable to create snapshot $runningVar@$MDS_SNAPNAME" - zfs clone "$runningVar@$MDS_SNAPNAME" "$runningVar/$MDS_SNAPNAME" || - die "unable to create $runningVar/$MDS_SNAPNAME" - mount -F zfs "$runningVar/$MDS_SNAPNAME" "$root/var/delphix" || - die "unable to mount $root/var/delphix" -} - -function cleanup_datasets() { - if [[ -n "$runningVar" ]]; then - zfs destroy -R "$runningVar@$MDS_SNAPNAME" - fi - umount -f "$root" - rmdir "$root" -} - -function cleanup_postgres() { - "$root$DX_MANAGE_PG" stop -s $MDS_SNAPNAME || - echo "failed to stop postgres" - "$root$DX_MANAGE_PG" cleanup -s $MDS_SNAPNAME || - echo "failed to clean up postgres" -} - -function cleanup_masking() { - if [[ -n "$masking_temp_dir" ]]; then - rm -r "$masking_temp_dir" - fi -} - -function run_upgrade_verify() { - local output=$1 - local format=$2 - local locale=$3 - local upgrade_verify_opts - $disable_consistent_mds_zfs_data_util && - upgrade_verify_opts="-disableConsistentMdsZfsDataUtil" - local progress_low=$4 - local progress_high=$5 - local debug_mode - debug_mode=$(svcprop -p delphix/debug $MGMT_FMRI) - - local jar=$root$UPGRADE_VERIFY_JAR - $java -Dlog.dir=$LOG_DIR -Dmdsverify=true -Ddelphix.debug="$debug_mode" \ - -DosMigration=true -jar "$jar" -d "$output" -f "$format" \ - -l "$locale" -v "$version" -root "$root" "$upgrade_verify_opts" \ - -droot "$root" -pl "$progress_low" -ph "$progress_high" || - die "upgrade verification failed" -} - -function test_masking() { - masking_temp_dir=$(mktemp -d) || - die "unable to make temporary directory" - - tar xof "$root/opt/delphix/masking/resources/upgrade-verification.tar" \ - -C "$masking_temp_dir" || - die "failed to extract masking upgrade verification tool" - - local verify_tool_dir=$masking_temp_dir/upgrade-verification - local new_conf=$root/opt/delphix/masking/conf/ - local overrides=/var/delphix/dmsuite/overrides/ - - # - # Run the verification tool against the new version of the .properties - # files in conf/ and against override.properties, to ensure that - # override.properties only overrides supported properties. - # - local cp="$verify_tool_dir/*:$new_conf:$overrides" - $java -cp "$cp" com.dmsuite.VerifyProperties || - die "verification of overrides failed" -} - -disable_consistent_mds_zfs_data_util=false -while getopts ':v:o:l:f:d' c; do - case "$c" in - f) format=$OPTARG ;; - l) locale=$OPTARG ;; - o) output=$OPTARG ;; - v) version=$OPTARG ;; - d) disable_consistent_mds_zfs_data_util=true ;; - *) usage ;; - esac -done - -[[ -n $version ]] || usage - -# Where root is or will be mounted. Set in mount_datasets. -root= -# The path of the running dataset mounted at /var/delphix. Set in mount_datasets. -runningVar= -# Temporary diresctory used by the masking checks. Set in test_masking. -masking_temp_dir= -# The installation of java to use when executing our jars. -java=/opt/jdk/bin/java - -report_progress_inc 0 "preparing for verification" -mount_datasets "$version" -report_progress_inc 20 "running upgrade checks" -run_upgrade_verify "$output" "$format" "$locale" 20 95 - -DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source -__trigger_unset_stress_option "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION" - -test_masking -report_progress_inc 95 "cleaning up post-verification" -cleanup -report_progress_inc 100 "done" -exit 0 diff --git a/live-build/misc/migration-scripts/etc_system_whitelist b/live-build/misc/migration-scripts/etc_system_whitelist deleted file mode 100644 index 5f9b3f77..00000000 --- a/live-build/misc/migration-scripts/etc_system_whitelist +++ /dev/null @@ -1,10 +0,0 @@ -sd:bypass_mode_sense_for_geometry -zfs_recover -zfs_vdev_async_read_max_active -zfs_vdev_async_read_min_active -zfs_vdev_async_write_max_active -zfs_vdev_async_write_min_active -zfs_vdev_sync_read_max_active -zfs_vdev_sync_read_min_active -zfs_vdev_sync_write_max_active -zfs_vdev_sync_write_min_active diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 04d1ab7c..991d67b3 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -94,7 +94,6 @@ if [[ "$RUN_TYPE" == "$ALL_RUN_TYPE" || "$RUN_TYPE" == "$VM_RUN_TYPE" ]]; then fi cp -r "$TOP/live-build/variants/$APPLIANCE_VARIANT/ansible" "$build_dir" -cp -r "$TOP/live-build/misc/migration-scripts" "$build_dir" cd "$build_dir" @@ -256,7 +255,7 @@ esac # user (e.g. other software); this is most useful when multiple variants # are built via a single call to "make" (e.g. using the "all" target). # -for ext in debs.tar.gz migration.tar.gz $vm_artifact_ext; do +for ext in debs.tar.gz $vm_artifact_ext; do if [[ -f "$ARTIFACT_NAME.$ext" ]]; then mv "$ARTIFACT_NAME.$ext" "$TOP/live-build/build/artifacts/" fi From 57ce711229a6c14621132426e351771aa2f34550 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 12 Jul 2021 13:53:19 -0700 Subject: [PATCH 083/197] DLPX-76433 [Backport of DLPX-76376 to 6.0.10.0] rootfs property "com.delphix:hotfix-version" incorrect after rollback of hotfix (#573) --- upgrade/upgrade-scripts/common.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index af027af8..e4f8efb2 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -223,7 +223,13 @@ function copy_optional_dataset_property() { local DST_DATASET="$3" local PROP_VALUE - PROP_VALUE=$(zfs get -Hpo value "$PROP_NAME" "$SRC_DATASET") + # + # Note, we only want to copy the dataset property when it's a + # local value, rather than a potentially inherited value. Thus, + # we use "-s local" to acheive this; i.e. with that set, if the + # value is not local, no value will be returned. + # + PROP_VALUE=$(zfs get -s local -Hpo value "$PROP_NAME" "$SRC_DATASET") # # Unlike the "copy_required_dataset_property" function, if the From 0194606e800eef04c2a261e32eb74cbe0516054c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 13 Jul 2021 11:42:12 -0400 Subject: [PATCH 084/197] DLPX-76478 [Backport of DLPX-71683 to 6.0.10.0] Generate new buildinfo package using metadata from combined packages (#493) (#572) --- .../appliance-build.bootstrap/tasks/main.yml | 1 + build-info/debian/compat | 1 + build-info/debian/control | 25 ++++ build-info/debian/copyright | 18 +++ build-info/debian/install | 1 + build-info/debian/rules | 19 +++ build-info/debian/source/format | 1 + scripts/build-ancillary-repository.sh | 7 ++ scripts/create-build-info-package.sh | 113 ++++++++++++++++++ 9 files changed, 186 insertions(+) create mode 100644 build-info/debian/compat create mode 100644 build-info/debian/control create mode 100644 build-info/debian/copyright create mode 100644 build-info/debian/install create mode 100755 build-info/debian/rules create mode 100644 build-info/debian/source/format create mode 100755 scripts/create-build-info-package.sh diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 9b067a02..a2b2fbb4 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -25,6 +25,7 @@ - awscli - bc - coreutils + - devscripts - equivs - gdisk - git diff --git a/build-info/debian/compat b/build-info/debian/compat new file mode 100644 index 00000000..f599e28b --- /dev/null +++ b/build-info/debian/compat @@ -0,0 +1 @@ +10 diff --git a/build-info/debian/control b/build-info/debian/control new file mode 100644 index 00000000..cdacd76a --- /dev/null +++ b/build-info/debian/control @@ -0,0 +1,25 @@ +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +Source: delphix-build-info +Section: misc +Priority: optional +Maintainer: Delphix Engineering +Build-Depends: debhelper (>= 10), devscripts +Standards-Version: 4.1.2 + +Package: delphix-build-info +Architecture: all +Description: Provides build metadata for packages built by Delphix. diff --git a/build-info/debian/copyright b/build-info/debian/copyright new file mode 100644 index 00000000..b4986c55 --- /dev/null +++ b/build-info/debian/copyright @@ -0,0 +1,18 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Files: * +Copyright: 2021 Delphix +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache License, Version 2 + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/build-info/debian/install b/build-info/debian/install new file mode 100644 index 00000000..338285f1 --- /dev/null +++ b/build-info/debian/install @@ -0,0 +1 @@ +/var/delphix-buildinfo diff --git a/build-info/debian/rules b/build-info/debian/rules new file mode 100755 index 00000000..42f3b90d --- /dev/null +++ b/build-info/debian/rules @@ -0,0 +1,19 @@ +#!/usr/bin/make -f +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +%: + dh $@ diff --git a/build-info/debian/source/format b/build-info/debian/source/format new file mode 100644 index 00000000..89ae9db8 --- /dev/null +++ b/build-info/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index ae512f26..e216476f 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -111,6 +111,13 @@ mkdir -p "$WORK_DIRECTORY/artifacts" download_combined_packages_artifacts "$AWS_S3_URI_COMBINED_PACKAGES" \ "$WORK_DIRECTORY/artifacts" +# +# Create a delphix-build-info package from the build metadata of each +# package and of appliance-build itself and store it along with the other +# downloaded artifacts. +# +"$TOP"/scripts/create-build-info-package.sh "$WORK_DIRECTORY/artifacts" + # # Find all debs and put them into a directory that will be fed into Aptly. # diff --git a/scripts/create-build-info-package.sh b/scripts/create-build-info-package.sh new file mode 100755 index 00000000..b25cd70a --- /dev/null +++ b/scripts/create-build-info-package.sh @@ -0,0 +1,113 @@ +#!/bin/bash -ex +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script creates a new package that stores build metadata for packages +# built by Delphix. It takes one argument: the path where the +# combined-packages artifacts have been downloaded. +# + +. "${BASH_SOURCE%/*}/common.sh" + +if [[ -z "$TOP" ]]; then + echo "Must be run inside the git repsitory." + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "Must specify the path of the combined-packages artifacts." 1>&2 + exit 1 +fi + +PKG_DIR="$1" +[[ -d "$PKG_DIR/packages" ]] || + die "$PKG_DIR must be a path to combined-packages artifacts." + +# +# Build info files will be installed into /var/delphix-buildinfo on the +# Delphix appliance. +# +target="$TOP/build-info/var/delphix-buildinfo" +mkdir -p "$target" + +# +# Copy build metadata for each package +# +mkdir "$target/packages" +cd "$PKG_DIR/packages" +for pkg in */; do + pushd "$pkg" &>/dev/null + mkdir "$target/packages/$pkg" + for file in GIT_HASH BUILD_INFO PACKAGE_MIRROR_URL_MAIN PACKAGE_MIRROR_URL_SECONDARY metadata.json; do + [[ -f "$file" ]] && cp "$file" "$target/packages/$pkg/" + done + popd &>/dev/null +done + +cp "$PKG_DIR/COMPONENTS" "$target/packages/" +cp "$PKG_DIR/KERNEL_VERSIONS" "$target/packages/" + +# +# Generate build metadata for appliance-build +# +mkdir "$target/appliance-build" +cd "$TOP" +git rev-parse HEAD >"$target/appliance-build/GIT_HASH" + +function check_env() { + # + # When the job is ran manually for testing purposes, we do not expect + # all environment to be set, so skip the env check. + # + [[ -n "$JENKINS_URL" ]] || return 0 + + local val="${!1}" + [[ -n "$val" ]] || die "check_env: $1 must be non-empty" + return 0 +} + +check_env APPLIANCE_BUILD_GIT_URL +echo "$APPLIANCE_BUILD_GIT_URL" >"$target/appliance-build/GIT_URL" +check_env APPLIANCE_BUILD_GIT_BRANCH +echo "$APPLIANCE_BUILD_GIT_BRANCH" >"$target/appliance-build/GIT_BRANCH" +check_env DELPHIX_PACKAGE_MIRROR_MAIN +echo "$DELPHIX_PACKAGE_MIRROR_MAIN" >"$target/appliance-build/DELPHIX_PACKAGE_MIRROR_MAIN" +check_env DELPHIX_PACKAGE_MIRROR_SECONDARY +echo "$DELPHIX_PACKAGE_MIRROR_SECONDARY" >"$target/appliance-build/DELPHIX_PACKAGE_MIRROR_SECONDARY" +check_env AWS_S3_OUTPUT +echo "$AWS_S3_OUTPUT" >"$target/appliance-build/ARTIFACTS_S3_LOCATION" +check_env DELPHIX_APPLIANCE_VERSION +echo "$DELPHIX_APPLIANCE_VERSION" >"$target/appliance-build/DELPHIX_APPLIANCE_VERSION" + +# +# Build the package +# +cd "$TOP/build-info" +# +# We include some random characters in the version string to avoid collisions +# with other build-info packages built for different platforms but included in +# the same upgrade image. The timestamp is insufficient here since the other +# platforms are built in parallel. +# +rnd="$(uuidgen | tr -d '-' | fold -w 8 | head -n 1)" +version="1.0.0-delphix-$(date '+%Y.%m.%d.%H.%M.%S')-$rnd" +dch --create --package delphix-build-info -v "$version" \ + "Automatically generated changelog entry." + +dpkg-buildpackage -uc -us -b + +mv ../delphix-build-info*deb "$PKG_DIR/packages/" From d761972c1b208810290dbf89e8c8a1b962670b40 Mon Sep 17 00:00:00 2001 From: Manoj Joseph Date: Wed, 28 Jul 2021 20:50:21 -0700 Subject: [PATCH 085/197] Add rust to dev environment (#544) (#577) Co-authored-by: Paul Dagnelie --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index a24a1a96..e17f4c35 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -23,6 +23,7 @@ - alien - autoconf - build-essential + - cargo - cppcheck - curl - emacs-nox @@ -43,7 +44,9 @@ - mandoc - nfs-kernel-server - parted + - pkg-config - python-minimal + - rustc - shellcheck - targetcli-fb - unzip From 3548595e4dc83e925eb99f0325f2247d9bd8d250 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 30 Jul 2021 09:58:52 -0400 Subject: [PATCH 086/197] Backport a few changes from master to 6.0/stage (#579) * remove crash-python role from appliance-build Now that we can run crash-python outside the git repo from any path and we are halting future development on it, its role on the appliance build is no longer needed for developer VMs. Note that deleting that role uncovered a different bug where the minimal-common role in internal-dev is needed by the ldap role and it would always run before it because it was listed as a dependency under the crash python role. Now that the crash-python role is removed we re-add an old version of that configuration making the minimal-common role run before the ldap role. * Add "shell" subcommand to upgrade-container script (#420) This adds a new "shell" subcommand to the "upgrade-container" script, which is intended to enable users to obtain an interactive shell session within the specified container. Currently, the "run" subcommand can be used, while specifying the shell to run (e.g. /bin/bash), but since that does not specify the "--pty" option with "systemd-run", the terminal may not work correctly (e.g. keyboard input, pager output, etc). Now, when folks want to obtain an interactive shell, they should use this new "shell" subcommand, rather than "run". * Fix "upgrade-container shell" argument handling (#426) Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Prakash Surya --- .../meta/main.yml | 19 ------- .../tasks/main.yml | 51 ------------------- .../internal-dev/ansible/playbook.yml | 13 +++-- upgrade/upgrade-scripts/upgrade-container | 11 ++++ 4 files changed, 21 insertions(+), 73 deletions(-) delete mode 100644 live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml delete mode 100644 live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml diff --git a/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml b/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml deleted file mode 100644 index 870f3a70..00000000 --- a/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -dependencies: - - role: appliance-build.minimal-common diff --git a/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml deleted file mode 100644 index 4d3526fd..00000000 --- a/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- apt: - name: - - git - - python3-pyelftools - - python3-pip - state: present - register: result - until: result is not failed - retries: 3 - delay: 60 - -- pip: - name: mypy - state: present - register: result - until: result is not failed - retries: 3 - delay: 60 - -- git: - repo: "https://gitlab.delphix.com/os-platform/crash-python.git" - dest: "/export/home/delphix/crash-python" - version: "next" - accept_hostkey: yes - update: no - -- file: - path: "/export/home/delphix/{{ item }}" - owner: delphix - group: staff - state: directory - recurse: yes - with_items: - - crash-python diff --git a/live-build/variants/internal-dev/ansible/playbook.yml b/live-build/variants/internal-dev/ansible/playbook.yml index c2ecff0d..7946d852 100644 --- a/live-build/variants/internal-dev/ansible/playbook.yml +++ b/live-build/variants/internal-dev/ansible/playbook.yml @@ -21,7 +21,16 @@ ansible_python_interpreter: /usr/bin/python3 passwordless_ldap_sudo: true roles: - - appliance-build.crash-python-development + # + # In order for the local appliance user (e.g. delphix) to be created + # properly, we need to ensure we attempt to create this user before + # we configure the system to use LDAP. Once we enable LDAP, we'll + # fail to properly create the user. Thus, we need to ensure we apply + # this role (which will create the user) prior to applying the LDAP + # specific configuration/role (which we do below). + # + - appliance-build.minimal-common + - appliance-build.minimal-internal # # In order for LDAP to work properly, we need a specific ordering of the # values contained in the "passwd" line of the "/etc/nsswitch.conf" file. @@ -32,8 +41,6 @@ # roles later. # - appliance-build.delphix-ldap - - appliance-build.minimal-common - - appliance-build.minimal-internal - appliance-build.minimal-development - appliance-build.masking-common - appliance-build.masking-development diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index b7e9610a..1591d5d2 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -508,6 +508,10 @@ function run() { --quiet --wait --pipe -- "$@" } +function shell() { + machinectl shell "$CONTAINER" +} + function convert_to_rootfs() { # # We're relying on the "mountpoint" property for the "data" and @@ -843,6 +847,7 @@ function usage() { echo "$PREFIX_SPACES stop " echo "$PREFIX_SPACES destroy " echo "$PREFIX_SPACES run " + echo "$PREFIX_SPACES shell " echo "$PREFIX_SPACES upgrade " echo "$PREFIX_SPACES convert-to-rootfs " echo "$PREFIX_SPACES get-type " @@ -891,6 +896,12 @@ run) shift 2 run "$@" ;; +shell) + [[ $# -lt 2 ]] && usage "too few arguments specified" + CONTAINER="$2" + shift 2 + shell + ;; upgrade) [[ $# -lt 2 ]] && usage "too few arguments specified" [[ $# -gt 2 ]] && usage "too many arguments specified" From 3ba04d835df60c479813835bf452487af1f502c0 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Tue, 3 Aug 2021 11:15:25 -0700 Subject: [PATCH 087/197] DLPX-76770 [Backport of DLPX-68846] logic for determining device path for grub operations during upgrade is fragile (#581) --- upgrade/upgrade-scripts/rootfs-container | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/rootfs-container b/upgrade/upgrade-scripts/rootfs-container index f2af998f..9789531a 100755 --- a/upgrade/upgrade-scripts/rootfs-container +++ b/upgrade/upgrade-scripts/rootfs-container @@ -87,7 +87,17 @@ function get_bootloader_devices() { # zpool list -vH rpool | awk '! /rpool|mirror|replacing|spare/ {print $1}' | - sed 's/p\{0,1\}[0-9]*$//' + while read -r part; do + # + # If the rpool is not installed a parition, we throw + # an error. We expect this to never happen, and the + # calling code is likely untested in that case, so we + # throw an error rather than try to handle it. + # + [[ "$(lsblk --nodeps -no type "/dev/$part")" == "part" ]] || + die "rpool installed on full disk \"$part\"" + lsblk -no pkname "/dev/$part" + done } function set_bootfs_not_mounted_cleanup() { From 63cd72f3d9e042cd0a1f21c83c100d02fb65c243 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 3 Aug 2021 14:57:24 -0400 Subject: [PATCH 088/197] DLPX-76769 [Backport of DLPX-76694 to 6.0.10.0] Failure to build DelphixConnector on Delphix Engine, causing gui_sanity testing to fail (#580) --- .../tasks/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index 1add61b9..7a752d03 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -15,6 +15,19 @@ # --- +# +# pkg-config is necessary for building the DelphixConnector, which is built +# when running "ant all". +# +- apt: + name: + - pkg-config + state: present + register: result + until: result is not failed + retries: 3 + delay: 60 + - file: path: "/etc/systemd/system/delphix-mgmt.service.d" owner: root From 4e194afe5f9e2f0debedae525f4654865ee3f386 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 10 Aug 2021 13:31:06 -0700 Subject: [PATCH 089/197] DLPX-76293 Copy Hotfix Metadata Generation to appliance-build (#574) (#590) Co-authored-by: Matt Skinner --- build.gradle | 3 ++- .../configuration/81-upgrade-repository.binary | 14 ++++++++++++++ live-build/config/hooks/configuration/template.ctl | 2 +- scripts/create-build-info-package.sh | 2 ++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 85e0bcc5..178c5bd9 100644 --- a/build.gradle +++ b/build.gradle @@ -78,7 +78,8 @@ for (variant in allVariants) { } } -def shellScripts = fileTree("scripts") + +def shellScripts = fileTree("scripts").include({ details -> + details.file.getName().endsWith(".sh") }) + fileTree("live-build/config/hooks").include({ details -> details.file.canExecute() }) + diff --git a/live-build/config/hooks/configuration/81-upgrade-repository.binary b/live-build/config/hooks/configuration/81-upgrade-repository.binary index a1ce6b40..c871aac8 100755 --- a/live-build/config/hooks/configuration/81-upgrade-repository.binary +++ b/live-build/config/hooks/configuration/81-upgrade-repository.binary @@ -80,6 +80,20 @@ EOF # rename 's/\%3a/:/g' binary/packages/*.deb +# +# Copy over the generated hotfix metadata. +# +if [[ -z "$AWS_S3_HOTFIX_METADATA" ]]; then + touch hotfix_metadata +else + aws s3 cp --only-show-errors "$AWS_S3_HOTFIX_METADATA" hotfix_metadata +fi + +if [[ ! -f "hotfix_metadata" ]]; then + echo "Could not generate hotfix_metadata file." + exit 1 +fi + # # Now we need to create the "delphix-entire" metapackage that will # depend on all of the packages that have been installed in the binary diff --git a/live-build/config/hooks/configuration/template.ctl b/live-build/config/hooks/configuration/template.ctl index 2faa6336..27e913e4 100644 --- a/live-build/config/hooks/configuration/template.ctl +++ b/live-build/config/hooks/configuration/template.ctl @@ -6,7 +6,7 @@ Standards-Version: 3.9.2 Package: delphix-entire-@@PLATFORM@@ Provides: delphix-entire Version: @@VERSION@@ -Extra-Files: packages.list, variant +Extra-Files: packages.list, variant, hotfix_metadata Description: Entirety of Delphix Appliance This package depends on all of the packages that constitute the entirety of the Delphix Appliance. This set of packages provide the necessary tools to run diff --git a/scripts/create-build-info-package.sh b/scripts/create-build-info-package.sh index b25cd70a..eee73912 100755 --- a/scripts/create-build-info-package.sh +++ b/scripts/create-build-info-package.sh @@ -92,6 +92,8 @@ check_env AWS_S3_OUTPUT echo "$AWS_S3_OUTPUT" >"$target/appliance-build/ARTIFACTS_S3_LOCATION" check_env DELPHIX_APPLIANCE_VERSION echo "$DELPHIX_APPLIANCE_VERSION" >"$target/appliance-build/DELPHIX_APPLIANCE_VERSION" +check_env AWS_S3_HOTFIX_METADATA +echo "$AWS_S3_HOTFIX_METADATA" >"$target/appliance-build/HOTFIX_METDATA" # # Build the package From c8ca4c3e26f8f84fa213364f6a87b096fc2de4c1 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Wed, 11 Aug 2021 12:41:41 -0700 Subject: [PATCH 090/197] install-ansible-lint is failing (#587) (#593) Co-authored-by: Pavel Zakharov --- .github/scripts/install-ansible-lint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/scripts/install-ansible-lint.sh b/.github/scripts/install-ansible-lint.sh index c36c6568..79d78dcd 100755 --- a/.github/scripts/install-ansible-lint.sh +++ b/.github/scripts/install-ansible-lint.sh @@ -5,7 +5,6 @@ sudo apt-get install ansible git clone https://github.com/willthames/ansible-lint /opt/ansible-lint cd /opt/ansible-lint git checkout v3.4.21 -git branch -D master # # GitHub Actions exposes the GITHUB_ENV file that can be used to From cb6a599e7a7fe87ab7fcf17f665cc32627e89bd6 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Thu, 12 Aug 2021 00:07:47 -0700 Subject: [PATCH 091/197] DLPX-76893 [Backport of DLPX-76802 to 6.0.10.0] Starting Verification container enables IP forwarding on host (#592) --- upgrade/upgrade-scripts/upgrade-container | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 1591d5d2..af6f303d 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -443,6 +443,29 @@ function stop() { sleep 1 done + # + # Starting the upgrade container would have enabled ip forwarding + # on the host (and in turn disabled LRO). Reset these settings + # on a best effor basis. + # + sysctl -w net.ipv4.ip_forward=0 || warn "failed to disable ip port forwarding" + + # + # Find all the interfaces configured on the host and set + # lro on where possible. The sed command retrieves the names of the + # interfaces. The name of the virtual interfaces of containers could + # have an @ symbol followed by device name in the output of the ethtool + # and this needs to be trimmed off. + # + + for i in $(ip -br link | sed 's/^\([^ @]\+\).*/\1/'); do + fixed=$(ethtool -k "$i" | grep large-receive-offload | grep -i fixed) + if [[ -z "$fixed" ]]; then + echo "updating lro setting for nic $i" + ethtool -K "$i" lro on || warn "failed to set lro ON for nic '$i'" + fi + done + machinectl status "$CONTAINER" &>/dev/null && die "timeout waiting for container termination: '$CONTAINER'" From 07279f73b6f56b7bbe084bd972804efd27ddd9d1 Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Fri, 27 Aug 2021 09:15:15 -0700 Subject: [PATCH 092/197] Want variant-specific package lists for `live-build(7)` (#596) --- .../appliance-build.dcenter/tasks/main.yml | 37 ++--------------- .../package-lists/dcenter.list.chroot | 41 +++++++++++++++++++ scripts/run-live-build.sh | 21 +++++++--- 3 files changed, 61 insertions(+), 38 deletions(-) create mode 100644 live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 987e286d..352b7b53 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -15,39 +15,6 @@ # --- - -- apt: - name: - - adoptopenjdk-java8-jdk - - bind9 - - dnsutils - - git - - isc-dhcp-server - - libldap2-dev - - libsasl2-dev - - nfs-common-dbgsym - - nfs-kernel-server - - nfs-kernel-server-dbgsym - - python3 - - python3-dbg - - python3-dev - - python3-ldap - - python3-marshmallow - - python3-marshmallow-doc - - python3-pip - - python3-pyvmomi - - python3-six - - python3-tenacity - - python3-toml - - python3-venv - - targetcli-fb - - telnet - state: present - register: result - until: result is not failed - retries: 3 - delay: 60 - - git: repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' version: master @@ -55,6 +22,10 @@ accept_hostkey: yes update: no +- alternatives: + name: java + path: /usr/lib/jvm/adoptopenjdk-java8-jdk-amd64/bin/java + # # By default, ubuntu restricts directories where dhcpd and named # can operate. For dcenter, we maintain the dhcp configuration diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot new file mode 100644 index 00000000..9f03db53 --- /dev/null +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -0,0 +1,41 @@ +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +adoptopenjdk-java8-jdk +bind9 +dnsutils +git +isc-dhcp-server +libldap2-dev +libsasl2-dev +nfs-common-dbgsym +nfs-kernel-server +nfs-kernel-server-dbgsym +openjdk-11-jdk-headless +python3 +python3-dbg +python3-dev +python3-ldap +python3-marshmallow +python3-marshmallow-doc +python3-pip +python3-pyvmomi +python3-six +python3-tenacity +python3-toml +python3-venv +targetcli-fb +telnet diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 991d67b3..60678abf 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018, 2020 Delphix +# Copyright 2018, 2021 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -93,14 +93,25 @@ if [[ "$RUN_TYPE" == "$ALL_RUN_TYPE" || "$RUN_TYPE" == "$VM_RUN_TYPE" ]]; then cp -r "$TOP/live-build/config/hooks/$VM_RUN_TYPE/." "$build_dir/config/hooks" fi +sed "s/@@PLATFORM@@/$APPLIANCE_PLATFORM/" \ + <"$build_dir/config/package-lists/delphix-platform.list.chroot.in" \ + >"$build_dir/config/package-lists/delphix-platform.list.chroot" + +if [[ -d "$TOP/live-build/variants/$APPLIANCE_VARIANT/package-lists" ]]; then + for list in "$TOP/live-build/variants/$APPLIANCE_VARIANT/package-lists/"*; do + [[ -f $list ]] || continue + if [[ -f "$build_dir/config/package-lists/$(basename "$list")" ]]; then + echo "Duplicate package list: $(basename "$list")" >&2 + exit 1 + fi + cp "$list" "$build_dir/config/package-lists" + done +fi + cp -r "$TOP/live-build/variants/$APPLIANCE_VARIANT/ansible" "$build_dir" cd "$build_dir" -sed "s/@@PLATFORM@@/$APPLIANCE_PLATFORM/" \ - config/package-lists/delphix-platform.list.chroot - # # The ancillary repository contains all of the first-party Delphix # packages that are required for live-build to operate properly. From 54a9461aedbac7044da2f626af766c03bb15ee84 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 27 Aug 2021 14:35:31 -0700 Subject: [PATCH 093/197] DLPX-77250 [Backport of DLPX-74852 to 6.0.11.0] Upgrade subsystem leaves behind stale entry in /etc/apt/sources.list (#602) --- upgrade/upgrade-scripts/execute | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 16e817d4..ecfb21b0 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -417,6 +417,14 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | sed 's/=/ /' | if [[ -f /etc/apt/sources.list.orig ]]; then mv /etc/apt/sources.list.orig /etc/apt/sources.list || die "failed to restore /etc/apt/sources.list" +else + # + # If "/etc/apt/sources.list.orig" does not exist, then it likely + # means the system didn't have a "/etc/apt/sources.list" file to + # begin with; so we restore that state by simply removing our + # dynamically generated "sources.list" file (generated earlier). + # + rm /etc/apt/sources.list fi # From 63f45838e3f8d538286a7936ce01f7780ab00230 Mon Sep 17 00:00:00 2001 From: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Date: Fri, 3 Sep 2021 09:57:44 -0700 Subject: [PATCH 094/197] TOOL-12103 [Backport of TOOL-12101 to 6.0/stage] ui-precommit fails with missing necessary HEADLESS chrome dependency (#604) --- .../appliance-build.unittest-internal/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index f5c9d47e..e079903d 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -15,6 +15,11 @@ # --- +# +# libxss1 and libgtk-3-0 are required dependencies for +# chromium-browser. They are missing in the chromium-browser +# package. Manually install them here. +# - apt: name: - adoptopenjdk-java8-jdk @@ -26,6 +31,7 @@ - python-minimal - chromium-browser - libxss1 + - libgtk-3-0 state: present - user: From 9ed212e3ed77f0cbebb723271fec29bab6420c4c Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Sep 2021 13:50:58 -0700 Subject: [PATCH 095/197] Use "delphix-rust" instead of Ubuntu's Rust packages (#606) --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index e17f4c35..ae60b685 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -23,9 +23,9 @@ - alien - autoconf - build-essential - - cargo - cppcheck - curl + - delphix-rust - emacs-nox - fakeroot - flake8 @@ -46,7 +46,6 @@ - parted - pkg-config - python-minimal - - rustc - shellcheck - targetcli-fb - unzip From fb7abc02594e16893ba376b180d9eb24f1aac9fc Mon Sep 17 00:00:00 2001 From: Raul Santelices Date: Wed, 15 Sep 2021 11:53:08 -0400 Subject: [PATCH 096/197] CP-5971 [Backport of CP-5946 to 6.0.11.0] Enable Java debug mode for sso app (#610) --- .../tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index 7a752d03..ed953e87 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -43,6 +43,21 @@ [Service] Environment=DLPX_DEBUG=true +- file: + path: "/etc/systemd/system/delphix-sso-app.service.d" + owner: root + group: root + state: directory + recurse: yes + +- copy: + dest: "/etc/systemd/system/delphix-sso-app.service.d/override.conf" + owner: root + group: root + content: | + [Service] + Environment=DLPX_DEBUG=true + - file: path: "/etc/systemd/system/delphix-postgres@.service.d" owner: root From ec0c4804873b9d19274234db041848b8a81b2c5f Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 28 Sep 2021 13:09:12 -0700 Subject: [PATCH 097/197] DLPX-77019 [Backport of DLPX-76998 to 6.0.11.0] CLONE - Starting Verification container enables IP forwarding on host (#611) --- upgrade/upgrade-scripts/upgrade-container | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index af6f303d..1924d37f 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -304,6 +304,15 @@ function create_upgrade_container() { # container can use ports without conflicting with ports that # are already in used by the host. # + # * VirtualEthernet: Since we're using private networking (see + # above), if we enable a virtual NIC, that will automatically + # result in IP forwarding being enabled, and LRO being disabled. + # We've found that with IP forwarding on and LRO off, network + # throughput can become up to 15% slower, adversely affecting + # workloads such as replication, DB sync, etc. Thus, we + # intentionally disable this setting, to avoid the network + # performance impact. + # # * Bind=/dev/zfs: We set this so that zfs/zpool/libzpool/etc. # is usable from within the container. # @@ -312,7 +321,7 @@ function create_upgrade_container() { PrivateUsers=no [Network] Private=yes - VirtualEthernet=yes + VirtualEthernet=no [Files] PrivateUsersChown=no Bind=/dev/zfs From 409e6863c630a20e290070d2d1538514c2019886 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 7 Oct 2021 15:18:39 -0400 Subject: [PATCH 098/197] DLPX-76940 [Backport of DLPX-76907 to 6.0.11.0] obsolete conf file cleanup logic incorrectly removes conf files that were moved to another package (#613) --- upgrade/upgrade-scripts/execute | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index ecfb21b0..801af01f 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -382,6 +382,17 @@ dpkg-query -Wf '${Conffiles}\n' | awk '$3 == "obsolete" {print $1}' | dpkg-query -W "$package" &>/dev/null || die "package '$package' is not installed" + # + # If the configuration file was moved to another package it + # will be listed for the original package as "obsolete" while + # also being listed as non-obsolete for the destination package. + # + if [[ $(dpkg-query -Wf '${Conffiles}\n' "$package" | + awk '$1 == "'"$file"'" {print $3}') != "obsolete" ]]; then + echo "configuration file '$file' has moved to package '$package'" + continue + fi + rm -f "$file" || die "failed to remove file '$file' of package '$package'" From c6a4f6a00c733b5bfb32aa74b4a103f92d84f982 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 5 Nov 2021 14:55:21 -0400 Subject: [PATCH 099/197] TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index ae60b685..a7d05917 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -59,12 +59,13 @@ delay: 60 - git: - repo: "https://github.com/delphix/zfs.git" + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/zfs.git" dest: "/export/home/delphix/zfs" version: master accept_hostkey: yes update: no + when: lookup('env', 'GITHUB_TOKEN') != '' - file: path: "/export/home/delphix/zfs" From 40386a6dff4dba384bb7aae4d4cf1b6eec7abcb9 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 5 Nov 2021 14:55:21 -0400 Subject: [PATCH 100/197] TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index ae60b685..a7d05917 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -59,12 +59,13 @@ delay: 60 - git: - repo: "https://github.com/delphix/zfs.git" + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/zfs.git" dest: "/export/home/delphix/zfs" version: master accept_hostkey: yes update: no + when: lookup('env', 'GITHUB_TOKEN') != '' - file: path: "/export/home/delphix/zfs" From f30b340541681dfebed3c2d9d37658365a1da388 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 13 Aug 2021 13:06:06 -0400 Subject: [PATCH 101/197] TOOL-12005 Enable ntp on dcenter-internal variant (#594) --- .../appliance-build.dcenter/tasks/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index 352b7b53..ae22ba52 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -62,3 +62,17 @@ - { regexp: '^RPCMOUNTDOPTS=', line: 'RPCMOUNTDOPTS="--num-threads=5 --manage-gids"' } - command: systemctl disable bind9.service isc-dhcp-server.service isc-dhcp-server6.service + +# +# delphix-platform installs ntp in a disabled state by default. +# We want to enable ntp to keep the time in sync on DCenter as clock skew +# can cause operational problems. +# +# For example, we run awscli on DCenter hosts, and some preliminary searching +# shows that aws s3 commands can return RequestTimeTooSkewed errors if there +# is clock skew. +# +# We also have cleanup jobs that run on DCenter hosts that rely on filesystem +# timestamps being accurate. +# +- command: systemctl enable ntp.service From 04e2830bda493ba5825275d6ade71551a4f3c61a Mon Sep 17 00:00:00 2001 From: Basil Crow Date: Fri, 27 Aug 2021 09:35:22 -0700 Subject: [PATCH 102/197] Use HTTPS URL for Artifactory (#598) --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 84207993..aac35717 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists distributionSha256Sum=7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d -distributionUrl=http://artifactory.delphix.com/artifactory/gradle-distributions/gradle-5.1-bin.zip +distributionUrl=https://artifactory.delphix.com/artifactory/gradle-distributions/gradle-5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 5df2e5052412e37935bf61c6f9cd4de75c4107ad Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 13 Oct 2021 15:30:54 -0400 Subject: [PATCH 103/197] DLPX-77385 Upgrade: all packages should be upgraded at the same time, remove exception for delphix-platform (#608) --- upgrade/upgrade-scripts/execute | 72 +++++++++------------------------ 1 file changed, 19 insertions(+), 53 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 801af01f..6daea1a5 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -175,59 +175,25 @@ DELPHIX_FLUENTD_IS_ENABLED=$(systemctl is-enabled delphix-fluentd.service) apt_get update || die "failed to update apt sources" # -# First make sure delphix-platform- is installed. This package -# is required by delphix-entire-, and would be installed even -# if we just relied on the packages.list.gz file log below. However, -# there are problems related to virtual packages that arise if we just -# did that. In particular, the following could happen (using AWS as the -# example for the platform on which the upgrade is being done): -# -# - We install the package requirements for "delphix-entire-aws" using -# the packages.list.gz file. -# -# - One of the packages in that list is the "delphix-virtualization" -# package, so it marks that package and its dependencies for -# installation. One of its dependencies is the virtual package -# "delphix-platform". -# -# - Apt then sees that there are multiple concrete packages which -# provide the "delphix-platform" virtual package. It chooses one of -# these arbitrarily, say "delphix-platform-kvm". -# -# - Apt marks all transitive dependencies of "delphix-platform-kvm" for -# installation, including, say, "linux-image-4.15.0-1028-kvm". -# -# - After processing the dependencies of "delphix-virtualization", Apt -# continues to process the other dependencies of "delphix-entire-aws", -# including "delphix-platform-aws". -# -# - Apt then sees that "delphix-platform-aws" conflicts with -# "delphix-platform-kvm", so it un-marks "delphix-platform-kvm" for -# installation, allowing "delphix-platform-aws" to satisfy the -# "delphix-virtualization" dependency on "delphix-platform". However, -# it does _not_ unmark all of the dependencies of -# "delphix-platform-kvm", so some unnecessary kvm-related packages -# end up being installed, such as "linux-image-4.15.0-1028-kvm" -# -# In short, we can end up with some unwanted packages installed because -# Apt marked and then unmarked some package for installation while -# trying to satisfy a dependency on a virtual package. -# -# To avoid this we install the delphix-platform- package that -# we want before installing delphix-entire- and it's required -# packages. If some delphix-platform package is already installed, that -# dependency is already satisfied and Apt doesn't need to choose an -# arbitrary one to satisfy the dependency. -# -# In general, we don't want upgrade to need know too much about the set -# of packages to be installed. However, it seems reasonable to make an -# exception for delphix-platform because delphix-platform is a -# fundemental piece of the architecture of the appliance. Among other -# things, it is the mechanism that we use to orchestrate the installation -# all of the platform-dependant bits of the appliance. -# -apt_get install -y "delphix-platform-$platform" || - die "failed to install delphix-platform" +# Currently, we need to run the delphix-platform's "postinst" packaging hook +# prior to the installation of the postgresql package. Further, right now, +# the postgresql package is pulled in as a dependency of the +# delphix-virtualization package. Thus, for the case of not-in-place upgrades, +# we accomplish this goal (of installing the delphix-platform package prior +# to the installation of the postgresql package) by first installing the +# delphix-platform package here, and then installing all other packages later. +# For the case of in-place upgrades, we can skip this step, as we assume the +# postgresql package is already installed, and simply upgrade all packages +# below. +# +# Note that we have the same issue during appliance-build, and it is solved +# by installing delphix-virtualization at a later stage of the build via +# ansible hooks, when the delphix-platform package has already been installed. +# +if ! dpkg-query -l "delphix-platform-$platform" &>/dev/null; then + apt_get install -y "delphix-platform-$platform" || + die "failed to install delphix-platform" +fi # # To enable the use of "autoremove" later in this script to remove all From bee1611e59354bf354111fa69fea67026c7ad369 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 19 Oct 2021 15:43:34 -0400 Subject: [PATCH 104/197] TOOL-12324 appliance-build: add a retry when unmount fails with EBUSY (#617) --- .../config/hooks/vm-artifacts/90-raw-disk-image.binary | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 2609306a..d353ad18 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -299,7 +299,12 @@ chroot "$DIRECTORY" grub-mkconfig -o /mnt/boot/grub/grub.cfg chroot "$DIRECTORY" umount /mnt for dir in /dev /proc /sys; do - umount -R "${DIRECTORY}${dir}" + for attempt in {1..5}; do + umount -R "${DIRECTORY}${dir}" && break + [[ "$attempt" == 5 ]] && die "Too many failed attempts, aborting." + echo "Attempt $attempt failed, trying again after a small nap." + sleep 10 + done done umount "$DIRECTORY/var/log" From 1ff83d56694aba876652363d582b4ba278bdf50d Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 19 Oct 2021 16:00:50 -0400 Subject: [PATCH 105/197] TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab (#619) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab * fixes --- .../ansible-roles/appliance-build.dcenter/tasks/main.yml | 9 +++++++++ .../appliance-build.masking-development/tasks/main.yml | 9 +++++++++ .../tasks/main.yml | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index ae22ba52..cac16958 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -15,12 +15,21 @@ # --- +# +# We are wrapping this in a retry block because the Gitlab server sometimes +# hangs up unexpectedly. We haven't seen this with GitHub, so we should +# probably remove the retry once we switch to GitHub. +# - git: repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' version: master dest: /opt/dcenter/lib/dcenter-gate accept_hostkey: yes update: no + retries: 3 + delay: 30 + register: result + until: result is not failed - alternatives: name: java diff --git a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml index 0e98eb17..b0711131 100644 --- a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml @@ -23,6 +23,11 @@ retries: 3 delay: 60 +# +# We are wrapping this in a retry block because the Gitlab server sometimes +# hangs up unexpectedly. We haven't seen this with GitHub, so we should +# probably remove the retry once we switch to GitHub. +# - git: repo: "{{ item.repo }}" dest: @@ -34,6 +39,10 @@ - { repo: 'https://gitlab.delphix.com/masking/dms-core-gate.git', version: master, dest: dms-core-gate } + retries: 3 + delay: 30 + register: result + until: result is not failed - file: path: "/export/home/delphix/{{ item }}" diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index ed953e87..2f93749e 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -73,6 +73,11 @@ [Service] Environment=DLPX_PG_DEBUG=true +# +# We are wrapping this in a retry block because the Gitlab server sometimes +# hangs up unexpectedly. We haven't seen this with GitHub, so we should +# probably remove the retry once we switch to GitHub. +# - git: repo: "{{ item.repo }}" dest: @@ -84,6 +89,10 @@ - { repo: 'https://gitlab.delphix.com/app/dlpx-app-gate.git', version: master, dest: dlpx-app-gate } + retries: 3 + delay: 30 + register: result + until: result is not failed - file: path: "/export/home/delphix/{{ item }}" From 7dc518ff03a83e493b324ab6247f1b517e7227a4 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 22 Oct 2021 12:47:29 -0400 Subject: [PATCH 106/197] DLPX-77878 Fix debootstrap arguments when creating not-in-place upgrade container (#616) --- upgrade/upgrade-scripts/upgrade-container | 30 +++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 1924d37f..e117c4b7 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -233,8 +233,34 @@ function create_upgrade_container() { # communicate and later run commands in the container with # "systemd-run". # - debootstrap --no-check-gpg \ - --components=delphix --include=systemd-container \ + # On Ubuntu 20.04, in-order to satisfy some package + # dependencies, we must set the variant to "minbase" and list + # "ntp" in the include list before "systemd-container". + # Setting the variant to minbase removes systemd from the + # packages being installed by default. Systemd will still + # be installed, but in a later pass, as a dependency of + # systemd-container. The reason we need to go through those + # hoops is the following: + # - systemd has a package dependency on a "time-daemon" + # virtual package, which is provided by either "ntp" or + # "systemd-timesyncd". + # - If a time-deamon is not already installed when + # installing systemd, then systemd will try to + # install systemd-timesyncd. + # - systemd-timesyncd is not available in our upgrade + # images because we install ntp and ntp conflicts with + # systemd-timesyncd. + # - Setting variant as minbase and list ntp before + # systemd-container in the include list allows debootstrap + # to install ntp before systemd, thus satisfying the + # package dependencies. + # Note that we do not run into those problems during live-build + # because debootstrap is already run with variant=minbase and + # systemd is not installed by debootstrap, but rather by + # delphix-platform which pulls both ntp and systemd. + # + debootstrap --no-check-gpg --variant=minbase \ + --components=delphix --include=ntp,systemd-container \ bionic "$DIRECTORY" "file://$IMAGE_PATH" 1>&2 || die "failed to debootstrap upgrade filesystem" From a14e536d231bfdb5da15cfbfa1ece2fe3686d58c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 22 Oct 2021 14:33:40 -0400 Subject: [PATCH 107/197] TOOL-12323 Remove libnss3-dbg package as it is missing on Ubuntu 20.04 (#618) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index e8b13d51..450ee3c7 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -48,7 +48,6 @@ - git - libcrypt-blowfish-dev - libcurl4-openssl-dev - - libnss3-dbg - libnss3-dev - libnss3-tools - libpam0g-dev From 5b5ee49b4edf96b69fe1f4bc789b771654f82f08 Mon Sep 17 00:00:00 2001 From: "Brandon T. Lim" Date: Tue, 26 Oct 2021 09:43:40 -0700 Subject: [PATCH 108/197] QI-3191 dlpx-qa-gate not installing on DCoLs as Jenkins agents (#623) Co-authored-by: Brandon Lim --- .../variants/internal-dcenter/package-lists/dcenter.list.chroot | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index 9f03db53..1f9264b2 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -39,3 +39,4 @@ python3-toml python3-venv targetcli-fb telnet +unixodbc-dev From 23c5632c41da1caa9265f4118fd560ab073b0fb7 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 22 Oct 2021 14:06:31 -0400 Subject: [PATCH 109/197] TOOL-12368 appliance-build: update some python2 package dependencies to python3 (#621) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 2 +- .../appliance-build.unittest-internal/tasks/main.yml | 2 +- .../appliance-build.zfsonlinux-development/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 450ee3c7..a442cb73 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -52,7 +52,7 @@ - libnss3-tools - libpam0g-dev - libssl-dev - - python-pip + - python3-pip state: present register: result until: result is not failed diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index e079903d..5b3b1324 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -28,7 +28,7 @@ - build-essential - docker.io - git - - python-minimal + - python3-minimal - chromium-browser - libxss1 - libgtk-3-0 diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index a7d05917..7e600184 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -45,7 +45,7 @@ - nfs-kernel-server - parted - pkg-config - - python-minimal + - python3-minimal - shellcheck - targetcli-fb - unzip From a0b19211f205e834e95815bd2f744f61914bf75c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 4 Nov 2021 18:19:52 -0400 Subject: [PATCH 110/197] DLPX-78255 ui-precommit fails due to missing Chrome dependency libnss3.so (#624) --- .../tasks/main.yml | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index 5b3b1324..bd48fabf 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -15,11 +15,6 @@ # --- -# -# libxss1 and libgtk-3-0 are required dependencies for -# chromium-browser. They are missing in the chromium-browser -# package. Manually install them here. -# - apt: name: - adoptopenjdk-java8-jdk @@ -29,9 +24,56 @@ - docker.io - git - python3-minimal - - chromium-browser - - libxss1 + state: present + +# +# The UI-precommit suite requires some chromium-browser dependendencies to be +# installed. However on 20.04 chromium is provided by a snap, and the +# "chromium-browser" package neither provides any dependencies nor the actual +# chromium browser, but rather directs users to install the chromium snap. +# While we could get chromium from a thrid party PPA, we do not actually need +# chromium for UI-precommit to work, but only some of the libraries that +# came as dependencies. As such we install here all the lib dependencies +# that were brought in by the chromium-browser package on 18.04 as per: +# https://packages.ubuntu.com/bionic/chromium-browser. +# +- apt: + name: + - libasound2 + - libatk-bridge2.0-0 + - libatk-bridge2.0-0 + - libatspi2.0-0 + - libc6 + - libcairo2 + - libcups2 + - libdbus-1-3 + - libdrm2 + - libexpat1 + - libgbm1 + - libgcc1 + - libgdk-pixbuf2.0-0 + - libglib2.0-0 - libgtk-3-0 + - libnspr4 + - libnss3 + - libpango-1.0-0 + - libpangocairo-1.0-0 + - libwayland-client0 + - libx11-6 + - libx11-xcb1 + - libxcb1 + - libxcomposite1 + - libxcursor1 + - libxdamage1 + - libxext6 + - libxfixes3 + - libxi6 + - libxkbcommon0 + - libxrandr2 + - libxrender1 + - libxshmfence1 + - libxss1 + - libxtst6 state: present - user: From b642813bb466498066945cd101c279724de0b808 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Wed, 10 Nov 2021 09:55:53 -0500 Subject: [PATCH 111/197] DLPX-78304 [Backport of DLPX-77901 to 6.0.12.0] Prevent services from being re-enabled on upgrade (#627) --- .../tasks/main.yml | 81 +++++++++++-- upgrade/upgrade-scripts/common.sh | 110 ++++++++++++++++++ upgrade/upgrade-scripts/execute | 45 ++++--- upgrade/upgrade-scripts/upgrade-container | 49 +------- 4 files changed, 207 insertions(+), 78 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml index 224ff921..e80afd0e 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml @@ -38,18 +38,83 @@ group: root state: link +# +# The section below deals with disabling and masking services that need to be +# disabled on initial install. Some of those services will remain disabled +# forever, while others could be enabled by the various applications running +# on the Delphix Appliance. +# +# Our approach is to mask services that need to be disabled. Masking a service +# instead of just disabling it has 2 advantages: +# 1. A service that is just disabled will not be started automatically by +# systemd on boot, however it can be started by another service if it has +# a certain set of dependencies on it (such as "Requires", "PartOf"). A +# service that is masked cannot be started in any case (although a +# service that is already running and has just been masked will not be +# stopped automatically). +# 2. A service that is disabled can get re-enabled by dpkg maintenance scripts +# when the package that provides that service is upgraded. A service that +# is masked will not be unmasked. +# +# While masking a service is sufficient, we also disable most of the services +# that we mask. This is not strictly necessary, but is done to remain +# consistent with how we did things in the past. Our current logic that deals +# with re-enabling those services will both unmask and enable # them. +# +# We have divided the disabling and masking of the services in multiple +# code blocks to group together services that are handled in the same way by +# the appliance. +# +# Note that if you want to modify this list make sure to also update +# the logic in fix_and_migrate_services() that is invoked during upgrade. +# You may also want to look at the logic that handles enabling disabling +# services in dlpx-app-gate. +# + +# +# The services in this section should always remain disabled & masked. +# +- name: Disable and mask services that should never run + shell: | + systemctl disable {{ item }} + systemctl mask {{ item }} + with_items: + - nginx.service + - postgresql.service + - systemd-timesyncd.service + +# +# The services in this section should be disabled & masked initially, but +# can be later dynamically enabled by the appliance. +# +- name: Disable and mask services that should not be running by default + shell: | + systemctl disable {{ item }} + systemctl mask {{ item }} + with_items: + - delphix-fluentd.service + - delphix-masking.service + - ntp.service + - snmpd.service + # # Because we want an NFSv4-only configuration out of the box, we need to mask # NFSv3 services so that they don't get automatically started at boot via # dependencies. The virtualization software is responsible for unmasking and # starting these services if NFSv3 needed at runtime. # -- name: Mask NFSv3 services - file: - src: "/dev/null" - dest: "/etc/systemd/system/{{ item }}" - state: link +- name: Mask NFSv3 services that should not be started automatically + command: "systemctl mask {{ item }}" + with_items: + - rpcbind.service + - rpcbind.socket + - rpc-statd.service + +# +# We disable docker. Instead of being started automatically by systemd, it +# gets started via a dependency of delphix-virtualization. +# +- name: Disable docker + command: "systemctl disable {{ item }}" with_items: - - rpc-statd.service - - rpcbind.service - - rpcbind.socket + - docker.service diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index e4f8efb2..0588a80a 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -362,3 +362,113 @@ function verify_upgrade_not_in_progress() { . "$UPDATE_DIR/upgrade.properties" &>/dev/null [[ -z "$UPGRADE_TYPE" ]] || die "upgrade currently in-progress" } + +function mask_service() { + local svc="$1" + local container="$2" + + # + # Note that masking should succeed even if service doesn't exist + # + if [[ -n "$container" ]]; then + chroot "/var/lib/machines/$container" systemctl mask "$svc" || + die "failed to mask '$svc' in container '$container'" + else + systemctl mask "$svc" || die "failed to mask '$svc'" + fi +} + +function is_svc_masked_or_disabled() { + local svc="$1" + + state=$(systemctl is-enabled "$svc") + if [[ "$state" == masked || "$state" == disabled ]]; then + return 0 + fi + + return 1 +} + +# +# This function has 2 tasks: +# 1. Fix/update the state of some services to be in line with what is expected +# in this version of the appliance. +# 2. If we are doing a not-in-place upgrade, then migrate the state of the +# services into the upgrade container. +# +# It can be called from 2 different contexts: +# 1. When creating upgrade container. In this case the container must be +# passed as first argument. +# 2. When executing the in-place upgrade. In this case the function takes no +# arguments. +# +function fix_and_migrate_services() { + local container="$1" + + # + # This function must be called from outside an upgrade container as it + # uses the state of the services on the running system as the source of + # truth. Since we want the logic in this script to apply both to an + # upgrade container and to the running system (in case of an in-place + # upgrade), we call it from two places: create_upgrade_container() and + # the execute script. The former will apply this logic on a container + # while the latter will apply this logic to the running system. + # + if systemd-detect-virt --container --quiet; then + echo "fix_and_migrate_services: should not run inside container" + return + fi + + # + # In versions prior to 6.0.13.0, snmpd.service was always enabled. + # Disable (mask) it here if we detect that it should have been disabled. + # + if compare_versions "$(get_current_version)" lt "6.0.13.0"; then + if [[ "$(systemctl is-enabled snmpd)" == enabled ]] && + ! grep -q "Delphix" /etc/snmp/snmpd.conf; then + mask_service snmpd "$container" + fi + fi + + # + # The services listed below are either permanently disabled or can be + # dynamically modified by the application(s) running on the appliance, + # so we need to ensure we migrate the state of these services when + # performing a not-in-place upgrade. Otherwise, we'd wind up with the + # default state of these services on initial install, which is to stay + # enabled and unmasked. + # + # If we are performing an in-place upgrade instead, then we want + # to make sure that the state of those services conforms to the new + # logic, which requires that the services are also masked when they + # are disabled. + # + # The reason we want to mask services instead of just disabling them + # is because when upgrading some of those packages, the services can + # be automatically enabled by postinst scripts; this is especially + # true for not-in-place upgrade, which creates a fresh debootstrap + # image on which the new packages are installed for the first time. + # + # Finally, some of the services that are masked may be both masked + # and disabled, while others would be only masked. This is okay + # given that masked services will not run whether they are enabled + # or disabled, and that the logic that unmasks them will also + # enable them. + # + while read -r svc; do + is_svc_masked_or_disabled "$svc" && + mask_service "$svc" "$container" + done <<-EOF + delphix-fluentd.service + delphix-masking.service + nfs-mountd.service + nginx.service + ntp.service + postgresql.service + rpc-statd.service + rpcbind.service + rpcbind.socket + snmpd.service + systemd-timesyncd.service + EOF +} diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 6daea1a5..cc7ef999 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -124,6 +124,22 @@ EOF start_stdout_redirect_to_system_log start_stderr_redirect_to_system_log +fix_and_migrate_services + +# +# Due to DLPX-77949, docker needs to be masked for the duration +# of the upgrade so that it does not get restarted automatically on +# upgrade, which would also force a restart of the delphix-mgmt +# service (since the latter has a dependency on docker.service), and +# thus interrupt the upgrade. +# +# Once the upgrade is done we restart delphix.target, which will +# attempt to restart both delphix-mgmt and docker, so docker +# needs to be unmasked before that point. As such, docker is +# unmasked at the end of this script. +# +systemctl mask docker.service + # # Older versions (i.e. the 6.0.0.0 release) of the "nfs-kernel-server" # package had "etab" file delivered as part of the package. Thus, when @@ -160,18 +176,6 @@ start_stderr_redirect_to_system_log [[ -e /var/lib/dpkg/info/nfs-kernel-server.list ]] && sed -i '/\/var\/lib\/nfs\/etab/d' /var/lib/dpkg/info/nfs-kernel-server.list -# -# Older versions (i.e. the 6.0.0.0 release) of the virtualization -# package would disable the "delphix-fluentd" service in that package's -# "prerm" package hook. This meant that if the service was enabled prior -# to the upgrade, it would be disabled after the upgrade. Since we can't -# easily stop this behavior on systems already running 6.0.0.0, we have -# to workaround this issue. Thus, before we upgrade the packages below, -# we check to see if this service is currently enabled, and will -# re-enable it after all packages have been upgraded. -# -DELPHIX_FLUENTD_IS_ENABLED=$(systemctl is-enabled delphix-fluentd.service) - apt_get update || die "failed to update apt sources" # @@ -303,17 +307,6 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | # apt_get autoremove --purge -y || die "autoremove after upgrade failed" -# -# As mentioned in a comment above, if the "delphix-fluentd" service is -# enabled prior to upgrading all of the packages, we need to ensure it -# remains enabled after upgrading all of the packages. Due to a bug in -# 6.0.0.0 this might not be the case, so we explicitly enable it here. -# -if [[ "$DELPHIX_FLUENTD_IS_ENABLED" == "enabled" ]]; then - systemctl enable "delphix-fluentd.service" || - die "failed to enable 'delphix-fluend.service'" -fi - # # Package configuration files are only automatically removed by the # package manager when the package that "owns" the file is "purged". @@ -375,6 +368,12 @@ dpkg-query -Wf '${Conffiles}\n' | awk '$3 == "obsolete" {print $1}' | die "failed to reinstall package '$package'" done || die "failed to remove obsolete package configuration files" +# +# Unmask docker, which was masked at the beginning of the upgrade due +# to DLPX-77949. +# +systemctl unmask docker.service + stop_stdout_redirect_to_system_log stop_stderr_redirect_to_system_log diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index e117c4b7..a5263ef2 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -421,6 +421,8 @@ function create_upgrade_container() { EOF die "failed to create container service override file" + fix_and_migrate_services "$CONTAINER" + echo "$CONTAINER" } @@ -639,33 +641,6 @@ function migrate_dir() { fi } -# -# Preserve the persistent service state of the service named as an argument. -# -function migrate_svc() { - local svc="$1" - local state - state=$(systemctl is-enabled "$svc") - - if [[ $state == "masked" ]]; then - chroot "/var/lib/machines/$CONTAINER" systemctl mask "$svc" - else - # - # The service may be masked by default, so always unmask before - # doing anything else. Otherwise, systemctl will ignore the new - # setting. - # - chroot "/var/lib/machines/$CONTAINER" systemctl unmask "$svc" - if systemctl is-enabled -q "$svc"; then - chroot "/var/lib/machines/$CONTAINER" systemctl \ - enable "$svc" - else - chroot "/var/lib/machines/$CONTAINER" systemctl \ - disable "$svc" - fi - fi -} - function migrate_configuration() { # # When performing a not-in-place upgrade, the root and delphix @@ -677,26 +652,6 @@ function migrate_configuration() { migrate_password_for_user delphix migrate_password_for_user root - # - # The services listed here can be dynamically modified by the - # application(s) running on the appliance, so we need to ensure - # we migrate the state of these services when performing a - # not-in-place upgrade. Otherwise, we'd wind up with the default - # state of these services after the upgrade, which could be - # different than the current state of these services. - # - while read -r svc; do - migrate_svc "$svc" - done <<-EOF - delphix-fluentd.service - delphix-masking.service - nfs-mountd.service - ntp.service - rpc-statd.service - rpcbind.service - rpcbind.socket - EOF - # # These files are generic OS files that are required for the # system to properly operate; these must be preserved acorss From a424acb95d83e3b4d9b3f2fc9adbabb6b060c9e2 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Thu, 11 Nov 2021 15:50:15 -0500 Subject: [PATCH 112/197] DLPX-78328 [Backport of DLPX-78275 to 6.0.12.0] Upgrade-verification is failing because /sys/module is not available in container (#629) --- upgrade/upgrade-scripts/upgrade-container | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index a5263ef2..07bdb2a3 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -342,6 +342,11 @@ function create_upgrade_container() { # * Bind=/dev/zfs: We set this so that zfs/zpool/libzpool/etc. # is usable from within the container. # + # * Bind=/sys/module: Starting with Ubuntu 20.04, /sys is mounted + # as tmpfs rather than sysfs and it seems like some sub-drectories + # are excluded by default, such as /sys/module. We need that + # directory to be present to run some commands (such as zfs). + # cat >"/etc/systemd/nspawn/$CONTAINER.nspawn" <<-EOF || [Exec] PrivateUsers=no @@ -351,6 +356,7 @@ function create_upgrade_container() { [Files] PrivateUsersChown=no Bind=/dev/zfs + Bind=/sys/module EOF die "failed to create container configuration file" From f2b13d2785f2e7de3e20209632736c79ba5776aa Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 1 Dec 2021 13:02:34 -0800 Subject: [PATCH 113/197] DLPX-78436 [Backport of DLPX-78371 to 6.0.12.0] Upgrade fails because some packages are unexpectedly autoremoved by apt (#635) Co-authored-by: Pavel Zakharov --- upgrade/upgrade-scripts/execute | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index cc7ef999..6aa34ad5 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -297,6 +297,10 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | xargs_apt_get install -y --allow-downgrades || die "failed to install packages listed in packages.list.gz file" +zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | + cut -d= -f1 | xargs apt-mark manual || + die "failed to mark as manual packages listed in packages.list.gz file" + # # After we've successfully installed the new packages, we need to remove # all packages that are no longer required. For example, if the old From 74543b0c46d2f9a8f85e1cb33dc199d02d447bb5 Mon Sep 17 00:00:00 2001 From: mrburke Date: Thu, 2 Dec 2021 20:48:15 -0800 Subject: [PATCH 114/197] DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index a442cb73..9c37b15b 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -40,6 +40,9 @@ # can move the build dependencies from the list below back into the # "internal-dev" playbook. # +- apt_repository: + repo: 'ppa:deadsnakes/ppa' + - apt: name: - ant @@ -53,6 +56,7 @@ - libpam0g-dev - libssl-dev - python3-pip + - python3.8 state: present register: result until: result is not failed From ccd9e2c9d97ec5593844b98395e9f827b87226fd Mon Sep 17 00:00:00 2001 From: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Date: Sun, 5 Dec 2021 13:50:36 -0500 Subject: [PATCH 115/197] TOOL-12541 [Backport of TOOL-12471 to 6.0.12.0] Remove Jenkins job references to devops-gate/master in appliance-build (#637) --- scripts/aptly-repo-from-debs.sh | 2 +- scripts/build-ancillary-repository.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/aptly-repo-from-debs.sh b/scripts/aptly-repo-from-debs.sh index f7628988..ea8caeaa 100755 --- a/scripts/aptly-repo-from-debs.sh +++ b/scripts/aptly-repo-from-debs.sh @@ -66,7 +66,7 @@ done # AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ "$AWS_S3_URI_COMBINED_PACKAGES" \ - "devops-gate/master/linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") + "linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") WORK_DIRECTORY=$(mktemp -d -p "$TOP/upgrade" tmp.pkgs.XXXXXXXXXX) diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index e216476f..6912c828 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -98,7 +98,7 @@ echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ "$AWS_S3_URI_COMBINED_PACKAGES" \ - "devops-gate/master/linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") + "linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") mkdir -p "$TOP/build" WORK_DIRECTORY=$(mktemp -d -p "$TOP/build" tmp.pkgs.XXXXXXXXXX) From 1d42c2645f5a44af740b450b9eb768fe558795ab Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 7 Dec 2021 13:43:13 -0800 Subject: [PATCH 116/197] Don't use "apt-get autoremove" anymore (#642) (#643) We've discovered that "apt-get autoremove" does not work for our needs. Specifically, packages may erroneously remain installed due to suggested and recommended package dependencies, as well as due to virtual package dependencies. While we could alleviate the issue w.r.t. suggested and recommended dependencies via configuration changes, there isn't a clear way to resolve the issue for virtual packages. Thus, the solution adopted in this change is to simply not use "apt-get automoremove" anymore, and instead, explicitly "apt-get purge" all package that we intend to be removed. This addresses the following two bugs: * DLPX-64201 determine how to remove old linux kernels * DLPX-78220 Old packages not removed after deferred upgrade --- upgrade/upgrade-scripts/execute | 55 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 6aa34ad5..874b54d7 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -200,32 +200,16 @@ if ! dpkg-query -l "delphix-platform-$platform" &>/dev/null; then fi # -# To enable the use of "autoremove" later in this script to remove all -# packages that are no longer required after the upgrade, we need to -# mark all packages currently installed as "auto" installed. This way, -# after explicitly installing all packages for the new "delphix-entire" -# version, the only "auto" installed packages still remaining on the -# system will be ones that aren't required by the new "delphix-entire". +# To accomplish the removal of packages that are no longer required +# after the upgrade, we need to mark all packages currently installed as +# "auto" installed. This way, after explicitly installing all packages +# for the new "delphix-entire" version, the only "auto" installed +# packages still remaining on the system will be ones that aren't +# required by the new "delphix-entire". # dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || die "failed to mark packages as 'auto' installed" -# -# While we marked all packages to "auto" above, so they'd be removed -# when they're no longer required, we want to prevent the removal of old -# kernel packages. This is so that we can boot into the old kernel(s) -# if we ever need to; e.g. after a failed kernel upgrade. We accomplish -# this by marking all "delphix-kernel-*" packages as "manual"-ly -# installed. The "delphix-kernel-*" package(s) will have package -# dependencies on the linux kernel package(s), and all other kernel -# packages that're required for that specific kernel (e.g. zfs kernel -# module package, connstat kernel module package, etc.). Thus, as long -# as the "delphix-kernel-*" package remains installed, all other -# dependent packages should also remain installed. -# -dpkg-query -Wf '${Package}\n' "delphix-kernel-*" | xargs apt-mark manual || - die "failed to mark 'delphix-kernel-*' packages as 'manual' installed" - # # In order to perform an upgrade of the Delphix appliance, we must first # install the "delphix-entire" package. This package will contain a @@ -302,14 +286,27 @@ zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | die "failed to mark as manual packages listed in packages.list.gz file" # -# After we've successfully installed the new packages, we need to remove -# all packages that are no longer required. For example, if the old -# "delphix-entire" version required a package, and the new version no -# longer requires that package, the commands run above will not remove -# the package. Thus, we need to run "autoremove" to ensure all packages -# that fall into this category are removed. +# After we've successfully installed the new packages and marked them +# all as manual-ly installed, we need to remove all packages that are no +# longer required. For example, if the old "delphix-entire" version +# required a package, and the new version no longer requires that +# package, the commands run above will not remove the package. +# +# Thus, we need to explicitly remove all these "leaked" packages here, +# and we rely on the fact that we apt-mark'ed all "old" packages as +# "auto" installed, and apt-mark'ed all "new" packages as "manual" +# installed. This way, we can easily determine which packages need to be +# removed, by simply removing all packages that're still labelled as +# "auto" installed. +# +# Additionally, we want to keep all packages relating to the currently +# running kernel. This way, in the event that the new kernel does not +# work properly, we'll still have the currently running kernel available +# to use as a fallback. # -apt_get autoremove --purge -y || die "autoremove after upgrade failed" +# shellcheck disable=SC2046 +apt_get purge -y $(apt-mark showauto | grep -v "$(uname -r)") || + die "failed to remove no-longer-needed packages" # # Package configuration files are only automatically removed by the From 80d70150d60bd5690c4a14a2e25db0364f95ac55 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Dec 2021 10:50:23 -0800 Subject: [PATCH 117/197] DLPX-78307 Upgrade tests to trunk failed (#647) (#649) --- upgrade/upgrade-scripts/upgrade-container | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 07bdb2a3..8484ef9e 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -295,7 +295,11 @@ function create_upgrade_container() { # is started, those NFS services will not run (no matter the # version of Delphix we're upgrading from). # - for svc in "nfs-mountd" "nfs-server" "rpc-statd" "rpc-statd-notify"; do + # DLPX-78307 - Similarly, we must disable "google-guest-agent". + # + for svc in \ + "nfs-mountd" "nfs-server" "rpc-statd" "rpc-statd-notify" \ + "google-guest-agent"; do [[ -e "$DIRECTORY/lib/systemd/system/$svc.service.d/override.conf" ]] && continue From e3898d3f048c1f1b3e7bb5aec1c580708538bd00 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 10 Dec 2021 15:17:24 -0800 Subject: [PATCH 118/197] DLPX-77974 Add minimum version property to root dataset (#620) (#650) --- build.gradle | 1 + live-build/build.gradle | 3 ++- .../hooks/vm-artifacts/90-raw-disk-image.binary | 6 ++++++ upgrade/upgrade-scripts/common.sh | 1 + upgrade/upgrade-scripts/execute | 4 ++++ upgrade/upgrade-scripts/upgrade-container | 14 ++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 178c5bd9..3ac2a611 100644 --- a/build.gradle +++ b/build.gradle @@ -53,6 +53,7 @@ for (variant in allVariants) { for (envVar in ["DELPHIX_PLATFORMS", "DELPHIX_HOTFIX_VERSION", + "DELPHIX_MINIMUM_VERSION", "AWS_S3_URI_LIVEBUILD_ARTIFACTS", "AWS_S3_URI_COMBINED_PACKAGES"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) diff --git a/live-build/build.gradle b/live-build/build.gradle index 4443c282..257e7aa4 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -105,7 +105,8 @@ for (variant in allVariants) { "DELPHIX_PACKAGE_MIRROR_MAIN", "DELPHIX_PACKAGE_MIRROR_SECONDARY", "DELPHIX_SIGNATURE_URL", - "DELPHIX_SIGNATURE_TOKEN"]) { + "DELPHIX_SIGNATURE_TOKEN", + "DELPHIX_MINIMUM_VERSION"]) { inputs.property(envVar, System.getenv(envVar)).optional(true) } diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index d353ad18..feb5706f 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -142,6 +142,12 @@ if [[ -n "$DELPHIX_HOTFIX_VERSION" ]]; then "rpool/ROOT/$FSNAME" fi +if [[ -n "$DELPHIX_MINIMUM_VERSION" ]]; then + zfs set \ + "com.delphix:minimum-version=$DELPHIX_MINIMUM_VERSION" \ + "rpool/ROOT/$FSNAME" +fi + zfs create \ -o canmount=noauto \ -o mountpoint=/ \ diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 0588a80a..efc17cc0 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -48,6 +48,7 @@ umask 0022 PROP_CURRENT_VERSION="com.delphix:current-version" PROP_INITIAL_VERSION="com.delphix:initial-version" PROP_HOTFIX_VERSION="com.delphix:hotfix-version" +PROP_MINIMUM_VERSION="com.delphix:minimum-version" # # To better enable root cause analysis of any upgrade failures, we diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 874b54d7..508355b4 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -445,6 +445,10 @@ zfs set "$PROP_CURRENT_VERSION=$VERSION" "$ROOTFS_CONTAINER" || die "failed to set property '$PROP_CURRENT_VERSION'" \ "to '$VERSION' for '$ROOTFS_CONTAINER'" +zfs set "$PROP_MINIMUM_VERSION=$MINIMUM_VERSION" "$ROOTFS_CONTAINER" || + die "failed to set property '$PROP_MINIMUM_VERSION'" \ + "to '$MINIMUM_VERSION' for '$ROOTFS_CONTAINER'" + if [[ -n "$HOTFIX" ]]; then zfs set "$PROP_HOTFIX_VERSION=$HOTFIX" "$ROOTFS_CONTAINER" || die "failed to set property '$PROP_HOTFIX_VERSION'" \ diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 8484ef9e..5b063ff8 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -119,6 +119,13 @@ function create_upgrade_container() { "$ROOTFS_DATASET" "$ROOTFS_DATASET@$SNAPSHOT_NAME" copy_optional_dataset_property "$PROP_HOTFIX_VERSION" \ "$ROOTFS_DATASET" "$ROOTFS_DATASET@$SNAPSHOT_NAME" + # + # The minimum version property was introduced in 6.0.12.0. + # + if compare_versions "$(get_current_version)" ge "6.0.12.0"; then + copy_required_dataset_property "$PROP_MINIMUM_VERSION" \ + "$ROOTFS_DATASET" "$ROOTFS_DATASET@$SNAPSHOT_NAME" + fi ;; rollback) # @@ -149,6 +156,13 @@ function create_upgrade_container() { "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" copy_optional_dataset_property "$PROP_HOTFIX_VERSION" \ "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" + # + # The minimum version property was introduced in 6.0.12.0. + # + if compare_versions "$(get_current_version)" ge "6.0.12.0"; then + copy_required_dataset_property "$PROP_MINIMUM_VERSION" \ + "$ROOTFS_DATASET@$SNAPSHOT_NAME" "rpool/ROOT/$CONTAINER" + fi zfs clone \ -o canmount=noauto \ From 026c778ac1f5eb9a818e229e4002bd9a4a3a7c32 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Sat, 8 Jan 2022 07:52:50 -0800 Subject: [PATCH 119/197] TOOL-12418 [Backport of TOOL-12369 to 6.0.13.0] appliance-build: changes for Ubuntu 20.04 (#653) --- live-build/auto/config | 2 +- live-build/config/archives/delphix-secondary-mirror.list.in | 5 +++-- live-build/config/archives/localhost.list | 2 +- .../appliance-build.virtualization-common/tasks/main.yml | 1 - scripts/aptly-repo-from-debs.sh | 2 +- scripts/aptly-repo-from-image-diff.sh | 2 +- scripts/build-ancillary-repository.sh | 2 +- scripts/run-live-build.sh | 2 +- upgrade/upgrade-scripts/execute | 2 +- upgrade/upgrade-scripts/upgrade-container | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/live-build/auto/config b/live-build/auto/config index 2dc7b897..ff65ae1c 100755 --- a/live-build/auto/config +++ b/live-build/auto/config @@ -35,7 +35,7 @@ lb config noauto \ --bootstrap-flavour minimal \ --chroot-filesystem none \ --architectures amd64 \ - --distribution bionic \ + --distribution focal \ --binary-images none \ --bootloader none \ --system normal \ diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 3d42434f..7de00f6f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,5 +14,6 @@ # limitations under the License. # -deb @@URL@@ bionic main multiverse universe -deb @@URL@@ bionic-updates main multiverse universe +deb @@URL@@ focal main multiverse universe +deb @@URL@@ focal-updates main multiverse universe +deb @@URL@@ focal-pgdg main diff --git a/live-build/config/archives/localhost.list b/live-build/config/archives/localhost.list index 7496bda3..7c633275 100644 --- a/live-build/config/archives/localhost.list +++ b/live-build/config/archives/localhost.list @@ -22,4 +22,4 @@ # used to serve the repository. # -deb [trusted=yes] http://localhost:8080 bionic main +deb [trusted=yes] http://localhost:8080 focal main diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml index e80afd0e..19432816 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml @@ -81,7 +81,6 @@ with_items: - nginx.service - postgresql.service - - systemd-timesyncd.service # # The services in this section should be disabled & masked initially, but diff --git a/scripts/aptly-repo-from-debs.sh b/scripts/aptly-repo-from-debs.sh index ea8caeaa..a2ea12ca 100755 --- a/scripts/aptly-repo-from-debs.sh +++ b/scripts/aptly-repo-from-debs.sh @@ -81,6 +81,6 @@ rm -rf "$WORK_DIRECTORY" # # Generate an Aptly/APT repository # -aptly repo create -distribution=bionic -component=delphix upgrade-repository +aptly repo create -distribution=focal -component=delphix upgrade-repository aptly repo add upgrade-repository debs aptly publish repo -skip-contents -skip-signing upgrade-repository diff --git a/scripts/aptly-repo-from-image-diff.sh b/scripts/aptly-repo-from-image-diff.sh index b987abb3..80891dd3 100755 --- a/scripts/aptly-repo-from-image-diff.sh +++ b/scripts/aptly-repo-from-image-diff.sh @@ -93,7 +93,7 @@ popd &>/dev/null || die "'popd' failed" # system (e.g. "upgrade-image-from-aptly-repo.sh"). # -aptly repo create -distribution=bionic -component=delphix upgrade-repository || +aptly repo create -distribution=focal -component=delphix upgrade-repository || die "failed to create repository: 'upgrade-repository'" aptly repo search image-a | xargs aptly repo copy image-a upgrade-repository || die "failed to copy packages to repository: 'upgrade-repository'" diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 6912c828..95e4d614 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -47,7 +47,7 @@ function build_ancillary_repository() { rm -rf "$HOME/.aptly" aptly repo create \ - -distribution=bionic -component=main ancillary-repository + -distribution=focal -component=main ancillary-repository aptly repo add ancillary-repository "$pkg_directory" aptly publish repo -skip-contents -skip-signing ancillary-repository diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 60678abf..79a6130a 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -127,7 +127,7 @@ APTLY_SERVE_PID=$! set +o errexit attempts=0 while ! curl --output /dev/null --silent --head --fail \ - "http://localhost:8080/dists/bionic/Release"; do + "http://localhost:8080/dists/focal/Release"; do ((attempts++)) if [[ $attempts -gt 30 ]]; then echo "Timed out waiting for ancillary repository." 1>&2 diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 508355b4..18a940dc 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -117,7 +117,7 @@ if [[ -f /etc/apt/sources.list ]]; then fi cat </etc/apt/sources.list || -deb [trusted=yes] file://$IMAGE_PATH bionic delphix +deb [trusted=yes] file://$IMAGE_PATH focal delphix EOF die "failed to configure apt sources" diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 5b063ff8..ef4c9806 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -275,7 +275,7 @@ function create_upgrade_container() { # debootstrap --no-check-gpg --variant=minbase \ --components=delphix --include=ntp,systemd-container \ - bionic "$DIRECTORY" "file://$IMAGE_PATH" 1>&2 || + focal "$DIRECTORY" "file://$IMAGE_PATH" 1>&2 || die "failed to debootstrap upgrade filesystem" # From 93e2819a6741965401c6ee3d38a2ff43163fabde Mon Sep 17 00:00:00 2001 From: Brad Lewis Date: Wed, 2 Feb 2022 10:54:45 -0700 Subject: [PATCH 120/197] CP-7160 [Backport of CP-6676 to 6.0.13.0] appliance-build changes to simplify the fluentd build (#655) --- live-build/config/archives/delphix-secondary-mirror.list.in | 2 +- .../appliance-build.virtualization-common/tasks/main.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 7de00f6f..07b7460f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,6 +14,6 @@ # limitations under the License. # -deb @@URL@@ focal main multiverse universe +deb @@URL@@ focal main multiverse universe contrib deb @@URL@@ focal-updates main multiverse universe deb @@URL@@ focal-pgdg main diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml index 19432816..8dd8d069 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml @@ -81,6 +81,7 @@ with_items: - nginx.service - postgresql.service + - td-agent.service # # The services in this section should be disabled & masked initially, but From fa056ba12d56c780838ca245fbbfbf3328a21d33 Mon Sep 17 00:00:00 2001 From: Brad Lewis Date: Mon, 7 Feb 2022 08:50:32 -0700 Subject: [PATCH 121/197] DLPX-79489 [Backport of DLPX-79367] td-agent service is not masked after upgrade (#659) --- upgrade/upgrade-scripts/common.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index efc17cc0..52ecca62 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -379,9 +379,14 @@ function mask_service() { fi } -function is_svc_masked_or_disabled() { +function is_svc_new_or_masked_or_disabled() { local svc="$1" + systemctl cat "$svc" &>/dev/null + if [ $? -eq 1 ]; then + return 0 + fi + state=$(systemctl is-enabled "$svc") if [[ "$state" == masked || "$state" == disabled ]]; then return 0 @@ -457,7 +462,7 @@ function fix_and_migrate_services() { # enable them. # while read -r svc; do - is_svc_masked_or_disabled "$svc" && + is_svc_new_or_masked_or_disabled "$svc" && mask_service "$svc" "$container" done <<-EOF delphix-fluentd.service From 978e6daf3aacb5e574c130b076fbf8789a39cef7 Mon Sep 17 00:00:00 2001 From: Brad Lewis Date: Mon, 7 Feb 2022 11:01:43 -0700 Subject: [PATCH 122/197] DLPX-79276 [Backport of DLPX-78736 to 6.0.13.0] Several test_api_and_cli_list_operation tests failed after stage to trunk upgrade (#660) --- upgrade/upgrade-scripts/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index 52ecca62..ba24ea65 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -476,5 +476,6 @@ function fix_and_migrate_services() { rpcbind.socket snmpd.service systemd-timesyncd.service + td-agent.service EOF } From ae1c7e0ec5a67bfa50d0f9d42c431cfd7a679667 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 22 Feb 2022 09:39:10 -0800 Subject: [PATCH 123/197] Rely on pre-installed "awscli" package (#661) (#662) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 9c37b15b..2130e6b9 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -46,7 +46,6 @@ - apt: name: - ant - - awscli - build-essential - git - libcrypt-blowfish-dev From 0d0cf2cee47ea192cd6f7f4c2e7d904654724c55 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 12 Apr 2022 12:58:13 -0700 Subject: [PATCH 124/197] DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670) --- .../appliance-build.unittest-internal/tasks/main.yml | 9 +++++++++ .../internal-dcenter/package-lists/dcenter.list.chroot | 1 + scripts/run-live-build.sh | 1 + 3 files changed, 11 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index bd48fabf..daaf3028 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -26,6 +26,15 @@ - python3-minimal state: present +# +# Prior to running the unit tests, the Jenkins automation will execute a +# script (enable-swap.sh) that formats a disk to use as a swap device. +# This script uses "sgdisk", so we need to ensure that's installed. +# +- apt: + name: gdisk + state: present + # # The UI-precommit suite requires some chromium-browser dependendencies to be # installed. However on 20.04 chromium is provided by a snap, and the diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index 1f9264b2..f0f33b9a 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -18,6 +18,7 @@ adoptopenjdk-java8-jdk bind9 dnsutils git +gnupg isc-dhcp-server libldap2-dev libsasl2-dev diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 79a6130a..47a88e71 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -208,6 +208,7 @@ fi set -o errexit lb config \ + --apt-recommends false \ --parent-mirror-bootstrap "$pkg_mirror_main" \ --parent-mirror-chroot "$pkg_mirror_main" \ --parent-mirror-chroot-security "$pkg_mirror_main" \ From 3415cf1bae8045891a20dcbc96a61678a2612100 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Tue, 12 Apr 2022 13:32:19 -0700 Subject: [PATCH 125/197] DLPX-79030 [Backport of DLPX-78464 to 6.0.14.0] Regression in appliance-build from DLPX-76293 (#669) --- .../config/hooks/configuration/81-upgrade-repository.binary | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/live-build/config/hooks/configuration/81-upgrade-repository.binary b/live-build/config/hooks/configuration/81-upgrade-repository.binary index c871aac8..6c37f7ec 100755 --- a/live-build/config/hooks/configuration/81-upgrade-repository.binary +++ b/live-build/config/hooks/configuration/81-upgrade-repository.binary @@ -83,11 +83,16 @@ rename 's/\%3a/:/g' binary/packages/*.deb # # Copy over the generated hotfix metadata. # +# Note: AWS_S3_HOTFIX_METADATA is the only variable that is allowed to +# be unset in this logic, thus the temporary `set -u`. +# +set +u if [[ -z "$AWS_S3_HOTFIX_METADATA" ]]; then touch hotfix_metadata else aws s3 cp --only-show-errors "$AWS_S3_HOTFIX_METADATA" hotfix_metadata fi +set -u if [[ ! -f "hotfix_metadata" ]]; then echo "Could not generate hotfix_metadata file." From cfc52f697748fcebd696bc3234b34ca40ba151dc Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Wed, 13 Apr 2022 15:37:57 -0400 Subject: [PATCH 126/197] Revert "DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670)" (#672) This reverts commit 0d0cf2cee47ea192cd6f7f4c2e7d904654724c55. --- .../appliance-build.unittest-internal/tasks/main.yml | 9 --------- .../internal-dcenter/package-lists/dcenter.list.chroot | 1 - scripts/run-live-build.sh | 1 - 3 files changed, 11 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index daaf3028..bd48fabf 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -26,15 +26,6 @@ - python3-minimal state: present -# -# Prior to running the unit tests, the Jenkins automation will execute a -# script (enable-swap.sh) that formats a disk to use as a swap device. -# This script uses "sgdisk", so we need to ensure that's installed. -# -- apt: - name: gdisk - state: present - # # The UI-precommit suite requires some chromium-browser dependendencies to be # installed. However on 20.04 chromium is provided by a snap, and the diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index f0f33b9a..1f9264b2 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -18,7 +18,6 @@ adoptopenjdk-java8-jdk bind9 dnsutils git -gnupg isc-dhcp-server libldap2-dev libsasl2-dev diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 47a88e71..79a6130a 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -208,7 +208,6 @@ fi set -o errexit lb config \ - --apt-recommends false \ --parent-mirror-bootstrap "$pkg_mirror_main" \ --parent-mirror-chroot "$pkg_mirror_main" \ --parent-mirror-chroot-security "$pkg_mirror_main" \ From 5c3fad85433b766d8327a015c2ff15b2f17be883 Mon Sep 17 00:00:00 2001 From: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Date: Fri, 15 Apr 2022 10:22:59 -0700 Subject: [PATCH 127/197] DLPX-80654 [Backport of DLPX-80114 to 6.0.14.0] Add list of installed linux packages for each platform variant during appliance build (#663) (#674) --- live-build/build.gradle | 3 +++ .../config/hooks/configuration/81-upgrade-repository.binary | 2 ++ scripts/run-live-build.sh | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/live-build/build.gradle b/live-build/build.gradle index 257e7aa4..f985a1fe 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -124,13 +124,16 @@ for (variant in allVariants) { switch (runType) { case upgradeArtifactsRunType: outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" + outputs.file "${buildDir}/artifacts/${variant}-${platform}.packages.list" break case vmArtifactsRunType: outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" + outputs.file "${buildDir}/artifacts/${variant}-${platform}.packages.list" break case allRunType: outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" + outputs.file "${buildDir}/artifacts/${variant}-${platform}.packages.list" break } diff --git a/live-build/config/hooks/configuration/81-upgrade-repository.binary b/live-build/config/hooks/configuration/81-upgrade-repository.binary index 6c37f7ec..de88a4b7 100755 --- a/live-build/config/hooks/configuration/81-upgrade-repository.binary +++ b/live-build/config/hooks/configuration/81-upgrade-repository.binary @@ -109,6 +109,8 @@ cp config/hooks/template.ctl delphix-entire.ctl # shellcheck disable=SC2016 chroot binary dpkg-query -Wf '${Package}=${Version}\n' | sort >packages.list +cp packages.list "$ARTIFACT_NAME.packages.list" + echo "$APPLIANCE_VARIANT" >variant test -n "$DELPHIX_APPLIANCE_VERSION" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 79a6130a..58be2d95 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -266,7 +266,7 @@ esac # user (e.g. other software); this is most useful when multiple variants # are built via a single call to "make" (e.g. using the "all" target). # -for ext in debs.tar.gz $vm_artifact_ext; do +for ext in debs.tar.gz $vm_artifact_ext packages.list; do if [[ -f "$ARTIFACT_NAME.$ext" ]]; then mv "$ARTIFACT_NAME.$ext" "$TOP/live-build/build/artifacts/" fi From 3707f920697f77508044a08ac9cdd9edd9e2110b Mon Sep 17 00:00:00 2001 From: George Wilson Date: Fri, 15 Apr 2022 18:22:58 -0400 Subject: [PATCH 128/197] TOOL-13514 [Backport of TOOL-13469 to 6.0.14.0] appliance-build changes to add telegraf support (#675) --- live-build/config/archives/delphix-secondary-mirror.list.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/config/archives/delphix-secondary-mirror.list.in b/live-build/config/archives/delphix-secondary-mirror.list.in index 07b7460f..9a0d395f 100644 --- a/live-build/config/archives/delphix-secondary-mirror.list.in +++ b/live-build/config/archives/delphix-secondary-mirror.list.in @@ -14,6 +14,6 @@ # limitations under the License. # -deb @@URL@@ focal main multiverse universe contrib +deb @@URL@@ focal main multiverse universe contrib stable deb @@URL@@ focal-updates main multiverse universe deb @@URL@@ focal-pgdg main From 6d9175e7c27489365466c08f080af55ba2d156b5 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 13 Apr 2022 13:50:28 -0700 Subject: [PATCH 129/197] Revert "DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636)" This reverts commit 74543b0c46d2f9a8f85e1cb33dc199d02d447bb5. --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 2130e6b9..c60935e2 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -40,9 +40,6 @@ # can move the build dependencies from the list below back into the # "internal-dev" playbook. # -- apt_repository: - repo: 'ppa:deadsnakes/ppa' - - apt: name: - ant From 7c9bec185c4071ae11c2da91af92ec1a60b6c360 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 12 Apr 2022 12:58:13 -0700 Subject: [PATCH 130/197] DLPX-80424 [Backport of DLPX-80287 to 6.0.15.0] remove unnecessary "recommended" packages from appliance --- .../appliance-build.unittest-internal/tasks/main.yml | 9 +++++++++ .../internal-dcenter/package-lists/dcenter.list.chroot | 1 + scripts/run-live-build.sh | 1 + 3 files changed, 11 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml index bd48fabf..daaf3028 100644 --- a/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml @@ -26,6 +26,15 @@ - python3-minimal state: present +# +# Prior to running the unit tests, the Jenkins automation will execute a +# script (enable-swap.sh) that formats a disk to use as a swap device. +# This script uses "sgdisk", so we need to ensure that's installed. +# +- apt: + name: gdisk + state: present + # # The UI-precommit suite requires some chromium-browser dependendencies to be # installed. However on 20.04 chromium is provided by a snap, and the diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index 1f9264b2..f0f33b9a 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -18,6 +18,7 @@ adoptopenjdk-java8-jdk bind9 dnsutils git +gnupg isc-dhcp-server libldap2-dev libsasl2-dev diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 58be2d95..cbc2bc71 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -208,6 +208,7 @@ fi set -o errexit lb config \ + --apt-recommends false \ --parent-mirror-bootstrap "$pkg_mirror_main" \ --parent-mirror-chroot "$pkg_mirror_main" \ --parent-mirror-chroot-security "$pkg_mirror_main" \ From 70ba63a6a7bb67e3add378894fcb99725693c81b Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 27 Apr 2022 14:55:51 -0700 Subject: [PATCH 131/197] DLPX-80917 [Backport of DLPX-80873 to 6.0.15.0] Upgrade verification gets stuck when upgrading from 6.0.13.0 -> 6.0.13.1 (#677) --- upgrade/upgrade-scripts/upgrade-container | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index ef4c9806..2924cc54 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -831,12 +831,11 @@ function do_upgrade_container_not_in_place() { } function get_type() { - local root origin + local origin - root=$(zfs list -Hpo name /var/lib/machines/"${CONTAINER}") - [[ -n "${root}" ]] || die "Failed to obtain root filesystem for container '${CONTAINER}'" - origin=$(zfs get -Hpo value origin "${root}") - [[ -n "${origin}" ]] || die "Failed to origin of the root filesystem '${root}' for container '${CONTAINER}'" + origin=$(zfs get -Hpo value origin "rpool/ROOT/${CONTAINER}/root") + [[ -n "${origin}" ]] || + die "failed to get origin for container '${CONTAINER}'" case "${origin}" in -) From 59a75d1a691ff033a079c00fde33deb5b9f14c7e Mon Sep 17 00:00:00 2001 From: George Wilson Date: Thu, 23 Jun 2022 14:48:40 -0400 Subject: [PATCH 132/197] CP-8412 [Backport of CP-8404 to 6.0.15.0] disable default telegraf service (#684) --- .../appliance-build.virtualization-common/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml index 8dd8d069..cc6ab39d 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml @@ -82,6 +82,7 @@ - nginx.service - postgresql.service - td-agent.service + - telegraf.service # # The services in this section should be disabled & masked initially, but From dbc960b4a4eead50d389a90644c025af53e44244 Mon Sep 17 00:00:00 2001 From: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:57:04 -0700 Subject: [PATCH 133/197] QA-35796 [Backport of QA-35550 to 6.0.15.0] Add nmap to qa-internal appliance builds (#679) --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index c60935e2..2a6c88ed 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -1,5 +1,5 @@ # -# Copyright 2018, 2021 Delphix +# Copyright 2018, 2022 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ - nftables - snmptrapd - ufw + - nmap state: present register: result until: result is not failed From ae1579ff4ecd9fb102bbb1009c72718871a01b6f Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Fri, 5 Aug 2022 23:06:43 +0530 Subject: [PATCH 134/197] DLPX-75209 -- Fix for upgrade scenarios --- upgrade/upgrade-scripts/execute | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 18a940dc..ac1192ac 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -17,6 +17,8 @@ . "${BASH_SOURCE%/*}/common.sh" +INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" + IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" @@ -28,6 +30,13 @@ function usage() { exit 2 } +function generate_interface_to_mac_address_map() { + ip -o link | grep -v "lo:" | tr -d ":" | awk '{print $2,$17}' | \ + tee "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || \ + die "Failed to generate map from interface names to mac addresses" +} + while getopts ':rlBfsp:' c; do case $c in r | l | B | f | s) ;; # LX-72: For now, silently ignore these. @@ -459,6 +468,50 @@ else "for '$ROOTFS_CONTAINER'" fi +# In AWS environments, changing instance type could result in generation of +# different network interface names compared to the existing ones. This mismatch +# causes our netplan file handler in the osadmin service to remove the existing +# entries in the netplan plan, because they are not to be found on the new instance. +# However, the mac address remains consistent and does not change with a change in +# the instance types. +# To handle this, from 6.0.16.0 onwards, we are switching to use mac address based +# interface names in the netplan file. To facilitate this change across upgrades from +# older verrsions to 6.0.16.0 and above, we create a map from the network interface +# names to their corresponding mac address, resulting in mac addrress based interface +# names. For wider context, see DLPX-75209. +# +# For deferred upgrades, we cannot know when the engine will be rebooted into the new +# image. The decision depends on the end-user. Hence we always create the map file +# irrespective of the upgrade type, to allow the upgrade finalize logic to copy the +# map file from the currently mounted root filesystem to the root filesystem of the +# target container. +# +# The map creation logic does not apply to engines being upgraded from 6.0.16.0 onwards. +# This is because the netplan file from 6.0.16.0 will by default be populated with mac +# address based interface names. This makes this logic redundant for upgrades from +# 6.0.16.0 and onwards. Hence, we must remove this logic as soon as we mark 6.0.16.0 as +# the minimum version supported release. + +# The container does not have visibility into the complete network namespace of the +# engine. Hence we need to create the map outside the container to ensure the presense +# of all the interfaces in the map +if ! systemd-detect-virt --container --quiet; then + platform_type=$(cat /var/lib/delphix-appliance/platform) + if [[ "$platform_type" == "aws" ]]; then + # we always create the map file anew to avoid using any stale versions of the same + if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then + rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || \ + die "Could not delete the existing interface to mac address map file" + fi + # While booting back to the original image post rollback, we want to reuse the original + # netplan file that consisted of the canonical interface names generated by the kernel. + # Hence, we skip creating the map file in rollback execution path + if [[ "$1" != "rollback" ]]; then + generate_interface_to_mac_address_map + fi + fi +fi + # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. From 0e446de4aa7a5bc8e5d903d2f07f152aa10320fe Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Tue, 9 Aug 2022 12:07:07 +0530 Subject: [PATCH 135/197] DLPX-75209 -- Fix for upgrade scenario --- upgrade/upgrade-scripts/execute | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 18a940dc..737a3f07 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -17,6 +17,8 @@ . "${BASH_SOURCE%/*}/common.sh" +INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" + IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" @@ -28,6 +30,25 @@ function usage() { exit 2 } +function generate_interface_to_mac_address_map() { + # we always create the map file anew to avoid using any stale versions of the same + if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then + > "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || \ + die "Could not delete the existing contents of the mac address map file" + fi + for i in $(ls -lart /sys/class/net | awk '{print $9}'); + do + interface_name="$i"; + if [ -e /sys/class/net/"$i"/address ] && \ + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/$i/address ; then + mac_address=$(cat /sys/class/net/"$i"/address | tr -d ":"); + echo $interface_name $mac_address >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + fi; + done + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || \ + die "Failed to generate map from interface names to mac addresses" +} + while getopts ':rlBfsp:' c; do case $c in r | l | B | f | s) ;; # LX-72: For now, silently ignore these. @@ -459,6 +480,42 @@ else "for '$ROOTFS_CONTAINER'" fi +# In AWS environments, changing instance type could result in generation of +# different network interface names compared to the existing ones. This mismatch +# causes our netplan file handler in the osadmin service to remove the existing +# entries in the netplan plan, because they are not to be found on the new instance. +# However, the mac address remains consistent and does not change with a change in +# the instance types. +# To handle this, from 6.0.16.0 onwards, we are switching to use mac address based +# interface names in the netplan file. To facilitate this change across upgrades from +# older verrsions to 6.0.16.0 and above, we create a map from the network interface +# names to their corresponding mac address, resulting in mac addrress based interface +# names. For wider context, see DLPX-75209. +# +# For deferred upgrades, we cannot know when the engine will be rebooted into the new +# image. The decision depends on the end-user. Hence we always create the map file +# irrespective of the upgrade type, to allow the upgrade finalize logic to copy the +# map file from the currently mounted root filesystem to the root filesystem of the +# target container. +# +# The map creation logic does not apply to engines being upgraded from 6.0.16.0 onwards. +# This is because the netplan file from 6.0.16.0 will by default be populated with mac +# address based interface names. This makes this logic redundant for upgrades from +# 6.0.16.0 and onwards. Hence, we must remove this logic as soon as we mark 6.0.16.0 as +# the minimum version supported release. + +# The container does not have visibility into the complete network namespace of the +# engine. Hence we need to create the map outside the container to ensure the presense +# of all the interfaces in the map +if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then + # While booting back to the original image post rollback, we want to reuse the original + # netplan file that consisted of the canonical interface names generated by the kernel. + # Hence, we skip creating the map file in rollback execution path + if [[ "$1" != "rollback" ]]; then + generate_interface_to_mac_address_map + fi +fi + # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. From 97b3e322267c0316573bab0c62189befa716c735 Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Wed, 24 Aug 2022 18:17:17 +0530 Subject: [PATCH 136/197] DLPX-75209 rebooting a Delphix Engine in AWS can leave the system with no network configuration --- upgrade/upgrade-scripts/execute | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 737a3f07..31c42fb4 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -33,19 +33,19 @@ function usage() { function generate_interface_to_mac_address_map() { # we always create the map file anew to avoid using any stale versions of the same if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then - > "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || \ + rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi for i in $(ls -lart /sys/class/net | awk '{print $9}'); do interface_name="$i"; - if [ -e /sys/class/net/"$i"/address ] && \ + if [ -e /sys/class/net/"$i"/address ] && grep -q -v -e "00:00:00:00:00:00" /sys/class/net/$i/address ; then mac_address=$(cat /sys/class/net/"$i"/address | tr -d ":"); echo $interface_name $mac_address >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" fi; done - [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || \ + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || die "Failed to generate map from interface names to mac addresses" } From ab2e04b989e304e87ee7516849d7d3e39b522151 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Wed, 24 Aug 2022 18:54:45 +0530 Subject: [PATCH 137/197] Update execute Update the execute script to follow guidelines of shfmt script. --- upgrade/upgrade-scripts/execute | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 31c42fb4..df27583f 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -31,22 +31,22 @@ function usage() { } function generate_interface_to_mac_address_map() { - # we always create the map file anew to avoid using any stale versions of the same - if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then - rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || - die "Could not delete the existing contents of the mac address map file" - fi - for i in $(ls -lart /sys/class/net | awk '{print $9}'); - do - interface_name="$i"; - if [ -e /sys/class/net/"$i"/address ] && - grep -q -v -e "00:00:00:00:00:00" /sys/class/net/$i/address ; then - mac_address=$(cat /sys/class/net/"$i"/address | tr -d ":"); - echo $interface_name $mac_address >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" - fi; - done - [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || - die "Failed to generate map from interface names to mac addresses" + # we always create the map file anew to avoid using any stale versions of the same + if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then + rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || + die "Could not delete the existing contents of the mac address map file" + fi + for i in $(ls -lart /sys/class/net | awk '{print $9}'); + do + interface_name="$i"; + if [ -e /sys/class/net/"$i"/address ] && + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/$i/address ; then + mac_address=$(cat /sys/class/net/"$i"/address | tr -d ":"); + echo $interface_name $mac_address >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + fi; + done + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || + die "Failed to generate map from interface names to mac addresses" } while getopts ':rlBfsp:' c; do @@ -508,12 +508,12 @@ fi # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then - # While booting back to the original image post rollback, we want to reuse the original - # netplan file that consisted of the canonical interface names generated by the kernel. - # Hence, we skip creating the map file in rollback execution path - if [[ "$1" != "rollback" ]]; then - generate_interface_to_mac_address_map - fi + # While booting back to the original image post rollback, we want to reuse the original + # netplan file that consisted of the canonical interface names generated by the kernel. + # Hence, we skip creating the map file in rollback execution path + if [[ "$1" != "rollback" ]]; then + generate_interface_to_mac_address_map + fi fi # From 2324b0ed7d491cacffc8d945bd94c6a6119b08b9 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 1 Sep 2022 10:30:21 +0530 Subject: [PATCH 138/197] Update execute Incorporate changes suggested by the shellcheck script --- upgrade/upgrade-scripts/execute | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index df27583f..f241b2da 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -36,13 +36,13 @@ function generate_interface_to_mac_address_map() { rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi - for i in $(ls -lart /sys/class/net | awk '{print $9}'); + for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do interface_name="$i"; if [ -e /sys/class/net/"$i"/address ] && - grep -q -v -e "00:00:00:00:00:00" /sys/class/net/$i/address ; then - mac_address=$(cat /sys/class/net/"$i"/address | tr -d ":"); - echo $interface_name $mac_address >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address ; then + mac_address=$(tr -d ":" < /sys/class/net/"$i"/address); + echo "$interface_name $mac_address" >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" fi; done [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || From c45555f3335ffc038d1b905eb11225288d26b9f5 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 1 Sep 2022 10:34:20 +0530 Subject: [PATCH 139/197] Update execute Incorporate changes suggested by the shfmt script. --- upgrade/upgrade-scripts/execute | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index f241b2da..48faf657 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -41,8 +41,8 @@ function generate_interface_to_mac_address_map() { interface_name="$i"; if [ -e /sys/class/net/"$i"/address ] && grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address ; then - mac_address=$(tr -d ":" < /sys/class/net/"$i"/address); - echo "$interface_name $mac_address" >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + mac_address=$(tr -d ":" < /sys/class/net/"$i"/address); + echo "$interface_name $mac_address" >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" fi; done [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || From 706918919402cfd17ee7950dd4880598d7c5149d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:04:40 +0530 Subject: [PATCH 140/197] Update execute Incorporate changes suggested by the shfmt script. --- upgrade/upgrade-scripts/execute | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 48faf657..9d5adac1 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -34,15 +34,14 @@ function generate_interface_to_mac_address_map() { # we always create the map file anew to avoid using any stale versions of the same if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || - die "Could not delete the existing contents of the mac address map file" + die "Could not delete the existing contents of the mac address map file" fi - for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); - do + for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do interface_name="$i"; if [ -e /sys/class/net/"$i"/address ] && - grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address ; then - mac_address=$(tr -d ":" < /sys/class/net/"$i"/address); - echo "$interface_name $mac_address" >> "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then + mac_address=$(tr -d ":" >"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" fi; done [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || From 118c60beb71cfda0bf8094f8379cc8618d8c3316 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:10:55 +0530 Subject: [PATCH 141/197] Update execute Incorporate changes suggested by the shfmt script. --- upgrade/upgrade-scripts/execute | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 9d5adac1..7a3f04f8 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -32,20 +32,20 @@ function usage() { function generate_interface_to_mac_address_map() { # we always create the map file anew to avoid using any stale versions of the same - if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then - rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || - die "Could not delete the existing contents of the mac address map file" - fi - for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do - interface_name="$i"; - if [ -e /sys/class/net/"$i"/address ] && - grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then - mac_address=$(tr -d ":" >"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" - fi; - done - [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || - die "Failed to generate map from interface names to mac addresses" + if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then + rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || + die "Could not delete the existing contents of the mac address map file" + fi + for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do + interface_name="$i" + if [ -e /sys/class/net/"$i"/address ] && + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then + mac_address=$(tr -d ":" >"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + fi + done + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || + die "Failed to generate map from interface names to mac addresses" } while getopts ':rlBfsp:' c; do @@ -508,11 +508,11 @@ fi # of all the interfaces in the map if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then # While booting back to the original image post rollback, we want to reuse the original - # netplan file that consisted of the canonical interface names generated by the kernel. - # Hence, we skip creating the map file in rollback execution path - if [[ "$1" != "rollback" ]]; then - generate_interface_to_mac_address_map - fi + # netplan file that consisted of the canonical interface names generated by the kernel. + # Hence, we skip creating the map file in rollback execution path + if [[ "$1" != "rollback" ]]; then + generate_interface_to_mac_address_map + fi fi # From 4f1423c2954daf7e7b5ae98467f250245351440e Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:20:51 +0530 Subject: [PATCH 142/197] 6.0/stage (#1) * [Backport of LB-209 to 6.0/release] Want to consume new mirror for production appliance build * [Backport to 6.0.0.0] Ensure that we are using secondary mirror (#371) * [Backport of LB-209 to 6.0/stage] Want to consume new mirror for production appliance build (#370) * [Backport to 6.0] Ensure that we are using secondary mirror (#372) * [Backport of LB-209 to 6.0/patch] Want to consume new mirror for production appliance build (#376) * [Backport to 6.0] Ensure that we are using secondary mirror (#377) * DLPX-66713 [Backport of Issue DLPX-66369 to 6.0.1.0] Using "upgrade-container stop" may not stop the running upgrade container (#375) * Make debug symbol packages (ddebs) available for installation (#378) * DLPX-66331 [Backport of Issue DLPX-66286 to 6.0.0.0] Interrupting a running upgrade verify can leave the engine not-upgradable (#385) * [Backport to 6.0.0.0] Set 'elevator=noop' GRUB option for first boot after migration (#386) * DLPX-66697 [Backport of Issue DLPX-66696 to 6.0.0.0] migration: add stress options in the pre-reboot dx scripts (#390) * DLPX-67336 [Backport of Issue DLPX-67184 to 6.0.0.0] migration: dx_apply should call dx_delete to cleanup previous state (#389) * DLPX-67355 [Backport of Issue DLPX-66957 to 6.0.0.0] hostname is not locally resolvable (#392) At boot time, applications will fail to resolve the local hostname to an IP address (i.e. the loopback address) until DNS servers are reachable, and once they are reachable, such lookups will result in external DNS requests, which is suboptimal given how often applications resolve the local hostname. These lookups should all be done locally without relying on external DNS servers. When applications resolve a hostname to an IP address, they use the NSS switch, which has a policy for how lookups should be done. On Delphix, the hosts entry for /etc/nsswitch.conf looks like this: hosts: files dns mymachines This means that any lookup will first consult local files (/etc/hosts), and if that fails, will go out to DNS. The /etc/hosts file is managed by cloud-init, and will contain the local hostname in cases where Delphix is running in the cloud. However, when Delphix is not in a cloud environment (e.g. on ESX), cloud-init does not update /etc/hosts with the local hostname (e.g. when it comes from a DHCP option). Lookups then end up going to DNS. The DNS configuration is in /etc/resolv.conf. Today, this file is a symbolic link to /run/systemd/resolve/resolv.conf, which in a DHCP environment, contains the set of DNS servers obtained via DHCP. As such, even requests to resolve the local hostname will end up going out to a remote DNS server. Ubuntu does, however, ship with a local caching DNS server that is built into systemd-resolved, and this caching DNS server has built-in support for resolving the local hostname without having to send such queries externally. Delphix doesn't use this because that would require that /etc/resolv.conf include a "localhost" DNS server so that DNS queries are made to the local DNS server. This can be accomplished by having /etc/resolv.conf be a symbolic link to /run/systemd/resolve/stub-resolv.conf. This resolver configuration points to the localhost server (systemd-resolved), which forwards queries that it wasn't able to satisfy from its local cache to external servers. This also happens to be the recommended default configuration according to the systemd-resolved.service man page. This fix simply changes the default symlink target for /etc/resolv.conf to be /run/systemd/resolve/stub-resolv.conf. * DLPX-67393 [Backport of Issue DLPX-67392 to 6.0.0.0] azure marketplace requires a 1mb unallocated region (#394) * DLPX-67946 [Backport of Issue DLPX-67884 to 6.0.0.0] need Hyper-V image (#403) * Stop using Docker when executing the build (#407) * Disable integration with Jenkins automation * Leverage Github Actions in favor of TravisCI (#398) This change adds the necessary files and logic to take advantage of GitHub Actions for running our automated tests. Currently we rely on TravisCI to do this, but we'd like to transition over to GitHub Actions, for the following reasons: 1. The Actions UI is more integrated with GitHub PRs 2. Best practices at Delphix suggests using Actions Additionally, this change removes integration with the "bors" bot. * Stop using Docker when executing the build (#404) * Modify propagation type of root dataset mount before mounting children (#402) (#408) Co-authored-by: John Gallagher * DLPX-68030 [Backport of Issue DLPX-67937 to 6.0.0.0] need OCI image (#410) * Remove (now) unused Docker related logic (#405) (#411) * [Backport to 6.0.1.0] Disable apt pipelining for the whole duration of live-build * [Backport to 6.0.0.1] Disable apt pipelining for the whole duration of live-build * Remove (now) unused Docker related logic (#405) (#418) * DLPX-67625 [Backport of Issue DLPX-67598 to 6.0.1.0] VDBs becomes inactive during some deferred upgrades from 6.0.1.0, similar to DLPX-66733 (#427) * DLPX-66981 Add verbose logging to the upgrade bash scripts (#397) * Log upgrade scripts' stdout and stderr via syslog (#414) This change builds on the prior commit 9ba7066. Now, in addition to logging the upgrade scripts' execution trace to syslog, we allow scripts to log the stdout and stderr of their commands to syslog as well. This new functionality is leveraged in the "execute" script, such that we keep a better log of what "apt-get" is doing when it's upgrading packages on the system. * Avoid removing "etab" file on upgrade (#421) * DLPX-66509 [Backport of Issue DLPX-66313 to 6.0.1.0] files previously installed into "/etc" by packages may not be removed on upgrade (#428) * Remove obsolete configuration files on upgrade (#425) * [Backport of DLPX-68440 to 6.0.1.0] remove server.conf and demo image from appliance-build (#429) * DLPX-68603 [Backport of DLPX-68595 to 6.0.1.0] delphix-fluentd service disabled during upgrade: fix for 6.0.0.0 (#436) * [Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0 * DLPX-68848 [Backport of DLPX-68832 to 6.0.2.0] DFE during trunk to trunk deferred upgrade, Failed to set-bootfs for container 'delphix.AvZ8Xle'. (#443) * Revert "[Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0" This reverts commit 7868fcdad6b620549fec13aee137c4429d46e92f. * DLPX-68826 [Backport of DLPX-68825 to 6.0.2.0] delphix-fluentd service disabled after not-in-place upgrade * DLPX-68769 [Backport of DLPX-68763 to 6.0.2.0] crashdump quota on rpool gets reset on upgrade (#453) * Backports of 4 commits to add 'upgrade finalize' to 6.0/stage (#457) * [Backport to 6.0.2.0] Add new "finalize" subcommand to upgrade script (#413) This change adds the "finalize" subcommand to the upgrade script, which is intended to be used to clean up any filesystems, snapshots, etc. that may have been generated during the upgrade process (e.g. for rollback). This is intended to be used to mark the finalization or completion of the upgrade, after which rollback will no longer be possible; e.g. $ sudo /var/dlpx-update/latest/upgrade deferred $ sudo /var/dlpx-update/latest/upgrade finalize * [Backport to 6.0.2.0] "rootfs-container delete" must process "log" dataset (#448) When "/var/log" was moved to a seperate dataset in commit 76a37744, we failed to properly adapt the "rootfs-container delete" script to account for this change. Since then, the "rootfs-container delete" script has likely been broken; currently, when I attempt to use this script, it fails with the following error: $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.xfXCbRc cannot destroy 'rpool/ROOT/delphix.xfXCbRc': filesystem has dependent clones use '-R' to destroy the following datasets: rpool/ROOT/delphix.iluoSLh/log rootfs-container: 'zfs destroy -r rpool/ROOT/delphix.xfXCbRc' failed The problem is, we're not properly handling the "log" dataset clones. The fix is simple, we need to add the "log" dataset to the list of datasets that we account for (i.e. the datasets that may need to be "zfs promote"-ed) when calling "rootfs-container delete". * [Backport to 6.0.2.0] Remove lingering snapshots after rollback event (#449) After performing a rollback of a failed not-in-place upgrade, if we remove the container associated with that failure (i.e. the "new" container), we will not remove the "container-delphix.XXXXXXX" snapshots used to generate that new container, and these snapshots will persist indefinitely. This change adds the necessary logic to the "rootfs-container delete" script to remove these snapshots, after removing the container itself. For example, without this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.Bs4VTPo $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU@container-delphix.Bs4VTPo 0B - 64K - rpool/ROOT/delphix.jDHoOtU/data 34.1M 44.3G 33.1M legacy rpool/ROOT/delphix.jDHoOtU/data@container-delphix.Bs4VTPo 1006K - 30.5M - rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/home@container-delphix.Bs4VTPo 151K - 11.8G - rpool/ROOT/delphix.jDHoOtU/log 28.7M 44.3G 27.0M legacy rpool/ROOT/delphix.jDHoOtU/log@container-delphix.Bs4VTPo 1.61M - 3.95M - rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / rpool/ROOT/delphix.jDHoOtU/root@container-delphix.Bs4VTPo 2.02M - 5.86G - And now, with this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.lJvMWgM $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU/data 34.5M 44.3G 34.5M legacy rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/log 27.1M 44.3G 27.1M legacy rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / Thus, with this change, the "container-delphix.XXXXXXX" snapshots are properly removed after calling "rootfs-container delete". * DLPX-69176 [Backport of DLPX-68157 to 6.0.2.0] Update 'finalize' logic to clean up snapshots older than current and previous Co-authored-by: Prakash Surya * Use "delphix-upgrade-verification" package in "verify-jar" script (#455) (#463) DLPX-64198 define and implement version info file for verification package DLPX-68919 Use New upgrade-verify.jar in appliance-build DLPX-69031 Make sure that the delphix-verification package is not installed post upgrade * correct branch.config * copy static DNS configuration for not-in-place upgrades (#470) * Remove unnecessary files from upgrade payload (#472) (#473) Currently, when we generate an upgrade image, we include the entire contents of "~/.aptly" in the image. It turns out, this causes us to include two identical copies of each package, one under "~/.aptly/pool" and another under "~/.aptly/public/pool". For the upgrade image, we only need the contents of the "~/.aptly/public" directory, so this change modifies the upgrade image generation logic, as well as the upgrade image application logic, to take this into account. As a result, this significantly reduces the size of the upgrade image (roughly a 50% reduction in size). * Checkout the master branch instead of projects/dx4linux on dcol. (#475) The projects/dx4linux branch in dcenter-gate is no longer used. All work is being done on master, and so that's the branch that needs to be checked out on deployed dcol instances. * dcenter image should increase mountd thread count [Backport of TOOL-9772] (#477) * backport tenacity requirement (part of TOOL-9418) (#478) * DLPX-71245 [Backport of DLPX-71244 to 6.0.3.0] fix for DLPX-69049 needs to also update grub for migrations (#480) * DLPX-70914 [Backport of DLPX-70835 to 6.0.4.0] disable usb-storage device module loading in delphix 6.x (5.3 migration part) (#483) * DLPX-71311 [Backport of DLPX-69447 to 6.0.4.0] Fixed hotfixes should be removed from /etc/hotfix during upgrade application (#484) * Backport: Install recovery env (#487) * Fix calls to "verify_upgrade_not_in_progress" (#462) (#486) The "verify_upgrade_not_in_progress" function only works correctly if the "UPGRADE_TYPE" variable is not set prior to the function being called. Unfortunately, in the two places we currently call that function, that variable will have already been set. This change fixes the issue by moving the calls to the function, such that we call it prior to setting the variable. Additionally, we add a check to the function to catch this error in usage earlier, and emit a more appropriate error message. Closes #461 * Fix "upgrade-container stop" return code on success (#433) (#434) This change fixes a regression introduced in acb60f4c. Without this change, calling "upgrade-container stop" will return an error code, even when the call successfully stops the container. With this change, when the container is stopped successfully, we'll now properly return 0. * fix branch.config * fix branch.config * Backports of Python package changes (#492) * Add py-six to DCenter appliance-build ansible role (#485) Co-authored-by: brandon.lim * Update DCenter Python package list (#488) - Add python-dbg and python-dev for use with GDB. - Remove python-requests, since we no longer depend on it as of TOOL-9913. - Add python-pyvmomi, because we eventually plan to migrate from PySphere to pyVmomi. Installing the package now enables us to begin preliminary testing with pyVmomi. - Add python-virtualenv, because we eventually plan to deploy our dependencies in a virtual environment. Installing the package now enables us to begin preliminary testing of virtual environments. - Add Python 3 versions of all Python 2 packages, because we eventually plan to migrate from Python 2 to Python 3. Installing Python 3 now enables us to begin preliminary testing with Python 3. Co-authored-by: brandon.lim * TOOL-10595 [Backport of TOOL-10160 to 6.0.6.0] linux-pkg rework: main appliance-build change (#496) * Github Action "::set-env" is no longer supported (#501) (#502) The Github Action "::set-env" command is no longer supported and results in failures when it is used. This change updates our scripts to remove the usage of that command, and use the "GITHUB_ENV" interface instead. Closes #495 * DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install (#499) * DLPX-73083 [Backport of DLPX-72429 to 6.0.6.0] ssh to admin CLI shows error message after force-not-in-place upgrade, permissions issue on /export dir (#504) * DLPX-72721 Cleanup old Illumos datasets on Linux to Linux upgrade (#500) (#505) * Backports of DCenter package updates (#508) Amends #488. Updates the DCenter package list to improve debuggability, adapt to recent changes in dcenter-gate, and prepare for future work. - Adding debug symbols via nfs-common-dbgsym and nfs-kernel-server-dbgsym to improve debuggability. - Removing python3-paramiko, as it is no longer needed when running dcenter-gate with pyVmomi. Note that Paramiko must remain installed for Python 2 until we drop support for PySphere (scheduled for later this month). - Removing virtualenv in favor of venv as in QI-1699. Note that we do not currently use a virtual environment, but we plan to start using one in future packaging and deployment work. We will likely be Python 3 only by the time that work is begun, so it makes more sense to use the native Python 3 venv than virtualenv. - Adding targetcli-fb in order to support prototyping dc volume. Also removes dangling references to dcenter.conf. * Revert "DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install" (#512) This reverts commit c26b447492dd39d87234f1dd2f9f8b49f3786874. * QA-25826 [Backport of QA-20893] Add the snmptrapd pkg to the delphix VM (#513) Co-authored-by: Neetha Menon * Backport hot-add to 6.0.6.0 (#516) * [Backport to 6.0.6.0] Enable generation of "sparse" upgrade image (#514) This change adds a new script that can be used to generate a "sparse" upgrade image. For example, given two input upgrade images, we can generate a sparse image like so: $ ./scripts/aptly-repo-from-image-diff.sh ../master/internal-dev.upgrade.tar ../6.0-release/internal-dev.upgrade.tar $ ./scripts/upgrade-image-from-aptly-repo.sh sparse-internal-dev $ ls -l build/artifacts/sparse-internal-dev.upgrade.tar -rw-r--r-- 1 delphix staff 5459210240 Jul 31 19:47 build/artifacts/sparse-internal-dev.upgrade.tar Additionally, we can inspect this new artifact and verify the set of packages it contains: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 199 And contrast this with the number of packages contained in both the "master" and "6.0-release" images: - master: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1042 - 6.0/release: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1080 Co-authored-by: Prakash Surya * DLPX-73378 [Backport of DLPX-73299 to 6.0.6.0] appliance-build: Fetch upgrade-verify package from combined-packages bundle (#515) * TOOL-10893 High disk space usage for audit logs threatens the success of future upgrades (#518) (#519) * CP-4470 [Backport of CP-4449 to 6.0.7.0] use pvscsi by default (#522) CP-4471 [Backport of CP-4451 to 6.0.7.0] Bump OVA hardware version to HWv11 * Backporting #523, #524, #525, and #526. (#527) * Backporting #523, #524, #525, and #526. (#528) * Fix branch.config * DLPX-71882 [Backport of DLPX-71833 to 6.0/stage] ui-precommit fails to start ChromeHeadless (#530) * Initial changes to support new hotfix workflow(s) (#507) (#531) * Support "hotfix-version" property on build and upgrade * Support hotfix version in upgrade image * Set hotfix version via "execute" script Co-authored-by: Prakash Surya * DLPX-75090 [Backport of DLPX-75089 to 6.0.8.0] Upgrade verification should not enable nfs services (#535) * DLPX-75232 [Backport of DLPX-75229 to 6.0.8.0] finalize() should not rely on properties in upgrade.properties (#539) * DLPX-75258 [Backport of DLPX-73923 to 6.0.9.0] Persist iSCSI initiator files during not-in-place upgrade (#542) * DLPX-75450 [Backport of DLPX-75347 to 6.0.9.0] old rootfs versions are not removed on upgrade in 6.0.8.0 (#545) * DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547) * DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549) * Don't remove "delphix-entire" when generating image diffs (#536) (#537) Currently, if we generate an upgrade image diff from two images that have the same delphix-entire package version, the resultant Aptly repository will not contain any delphix-entire package in it. This is problematic when that Aptly repository is later used with the script used to generate an upgrade image from that repository, as the script requires the repository to contain a "delphix-entire" package in it. Thus, to avoid this issue, this change modifies the image diff logic to ensure we never remove the "delphix-entire" package. * DLPX-75091 DLPX-75122 Make upgrade image prepare script hotfix aware (#543) (#550) DLPX-75091 Make upgrade image prepare script hotfix aware DLPX-75122 Use delphix version and hotfix in version.info in verify-jar script * Revert "DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549)" (#555) This reverts commit f6421aa1f2112ec17eb53737d079ff87f1e1a533. * Revert "DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547)" (#556) This reverts commit 26c960f1697c1cef7d66bae8326670094b9e206d. * DLPX-76243 management stack failed to come back after rollback upgrade (#558) * CP-5356 [Backport of CP-5334 to 6.0.9.0] Add ufw package to appliance-build (#559) * DLPX-76329 [Backport of DLPX-76328 to 6.0.9.0] upgrade script obtains hotfix version incorrectly (#564) In commit e7dd85e2 we introduced the `get_hotfix_version` function for obtaining the hotfix version of the currently running root filesystem. The problem is that in c4357a4e we added logic that intended to use this function, but instead tries to call the `get_current_hotfix` function, which doesn't exist. This leads us to never retreiving the hotfix version correctly via the `rollback` subcommand of the `upgrade` script. * DLPX-74045 adapt upgrade "execute" script to better handle new hotfix workflow (#520) (#568) Currently when applying a hotfix image generated by the new hotfix workflow being developed in CP-4190, nothing will happen when the upgrade is attempted. This is due to the "delphix-entire" package in the hotfix being installed, matching the version of the package that's already installed. Thus, the package manager chooses not to install the new packagecontained in the upgrade/hotfix image, since it thinks the same package is already installed, due to the two having the same version. We need to adapt the "execute" script that's used during upgrade, such that we always install the new "delphix-entire" package contained in the upgrade/hotfix image, even if the package versions are identical. * TOOL-11824 Harden against mistakes when running via Jenkins (#569) (#570) This change adds logic to cause a failure when certain environment variables are missing, and the build is being run via Jenkins. When environment variables are missing, we currently will provide default values, such that the build doesn't fail. These default values generally are not sufficient for anything besides allowing the build to complete (e.g. the resultant upgrade images may fail upgrade). Further, the main motivation for generating these default values, is to make it easier to run the build manually, and aid developer iteration. Thus, when it's clear that the build is being run via Jenkins, where these missing environment variables more likely point to an error or broken automation, than an intentional omission of these variables, it's better to report the error than set and use a default value. This way, we can more quickly detect and address the problem, than silently produce (potentially) incorrect or broken artifacts. * [Backport to 6.0] Remove code for building migration images (#571) Co-authored-by: John Gallagher * DLPX-76433 [Backport of DLPX-76376 to 6.0.10.0] rootfs property "com.delphix:hotfix-version" incorrect after rollback of hotfix (#573) * DLPX-76478 [Backport of DLPX-71683 to 6.0.10.0] Generate new buildinfo package using metadata from combined packages (#493) (#572) * Add rust to dev environment (#544) (#577) Co-authored-by: Paul Dagnelie * Backport a few changes from master to 6.0/stage (#579) * remove crash-python role from appliance-build Now that we can run crash-python outside the git repo from any path and we are halting future development on it, its role on the appliance build is no longer needed for developer VMs. Note that deleting that role uncovered a different bug where the minimal-common role in internal-dev is needed by the ldap role and it would always run before it because it was listed as a dependency under the crash python role. Now that the crash-python role is removed we re-add an old version of that configuration making the minimal-common role run before the ldap role. * Add "shell" subcommand to upgrade-container script (#420) This adds a new "shell" subcommand to the "upgrade-container" script, which is intended to enable users to obtain an interactive shell session within the specified container. Currently, the "run" subcommand can be used, while specifying the shell to run (e.g. /bin/bash), but since that does not specify the "--pty" option with "systemd-run", the terminal may not work correctly (e.g. keyboard input, pager output, etc). Now, when folks want to obtain an interactive shell, they should use this new "shell" subcommand, rather than "run". * Fix "upgrade-container shell" argument handling (#426) Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Prakash Surya * DLPX-76770 [Backport of DLPX-68846] logic for determining device path for grub operations during upgrade is fragile (#581) * DLPX-76769 [Backport of DLPX-76694 to 6.0.10.0] Failure to build DelphixConnector on Delphix Engine, causing gui_sanity testing to fail (#580) * DLPX-76293 Copy Hotfix Metadata Generation to appliance-build (#574) (#590) Co-authored-by: Matt Skinner * install-ansible-lint is failing (#587) (#593) Co-authored-by: Pavel Zakharov * DLPX-76893 [Backport of DLPX-76802 to 6.0.10.0] Starting Verification container enables IP forwarding on host (#592) * Want variant-specific package lists for `live-build(7)` (#596) * DLPX-77250 [Backport of DLPX-74852 to 6.0.11.0] Upgrade subsystem leaves behind stale entry in /etc/apt/sources.list (#602) * TOOL-12103 [Backport of TOOL-12101 to 6.0/stage] ui-precommit fails with missing necessary HEADLESS chrome dependency (#604) * Use "delphix-rust" instead of Ubuntu's Rust packages (#606) * CP-5971 [Backport of CP-5946 to 6.0.11.0] Enable Java debug mode for sso app (#610) * DLPX-77019 [Backport of DLPX-76998 to 6.0.11.0] CLONE - Starting Verification container enables IP forwarding on host (#611) * DLPX-76940 [Backport of DLPX-76907 to 6.0.11.0] obsolete conf file cleanup logic incorrectly removes conf files that were moved to another package (#613) * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12005 Enable ntp on dcenter-internal variant (#594) * Use HTTPS URL for Artifactory (#598) * DLPX-77385 Upgrade: all packages should be upgraded at the same time, remove exception for delphix-platform (#608) * TOOL-12324 appliance-build: add a retry when unmount fails with EBUSY (#617) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab (#619) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab * fixes * DLPX-77878 Fix debootstrap arguments when creating not-in-place upgrade container (#616) * TOOL-12323 Remove libnss3-dbg package as it is missing on Ubuntu 20.04 (#618) * QI-3191 dlpx-qa-gate not installing on DCoLs as Jenkins agents (#623) Co-authored-by: Brandon Lim * TOOL-12368 appliance-build: update some python2 package dependencies to python3 (#621) * DLPX-78255 ui-precommit fails due to missing Chrome dependency libnss3.so (#624) * DLPX-78304 [Backport of DLPX-77901 to 6.0.12.0] Prevent services from being re-enabled on upgrade (#627) * DLPX-78328 [Backport of DLPX-78275 to 6.0.12.0] Upgrade-verification is failing because /sys/module is not available in container (#629) * DLPX-78436 [Backport of DLPX-78371 to 6.0.12.0] Upgrade fails because some packages are unexpectedly autoremoved by apt (#635) Co-authored-by: Pavel Zakharov * DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636) * TOOL-12541 [Backport of TOOL-12471 to 6.0.12.0] Remove Jenkins job references to devops-gate/master in appliance-build (#637) * Don't use "apt-get autoremove" anymore (#642) (#643) We've discovered that "apt-get autoremove" does not work for our needs. Specifically, packages may erroneously remain installed due to suggested and recommended package dependencies, as well as due to virtual package dependencies. While we could alleviate the issue w.r.t. suggested and recommended dependencies via configuration changes, there isn't a clear way to resolve the issue for virtual packages. Thus, the solution adopted in this change is to simply not use "apt-get automoremove" anymore, and instead, explicitly "apt-get purge" all package that we intend to be removed. This addresses the following two bugs: * DLPX-64201 determine how to remove old linux kernels * DLPX-78220 Old packages not removed after deferred upgrade * DLPX-78307 Upgrade tests to trunk failed (#647) (#649) * DLPX-77974 Add minimum version property to root dataset (#620) (#650) * TOOL-12418 [Backport of TOOL-12369 to 6.0.13.0] appliance-build: changes for Ubuntu 20.04 (#653) * CP-7160 [Backport of CP-6676 to 6.0.13.0] appliance-build changes to simplify the fluentd build (#655) * DLPX-79489 [Backport of DLPX-79367] td-agent service is not masked after upgrade (#659) * DLPX-79276 [Backport of DLPX-78736 to 6.0.13.0] Several test_api_and_cli_list_operation tests failed after stage to trunk upgrade (#660) * Rely on pre-installed "awscli" package (#661) (#662) * DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670) * DLPX-79030 [Backport of DLPX-78464 to 6.0.14.0] Regression in appliance-build from DLPX-76293 (#669) * Revert "DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670)" (#672) This reverts commit 0d0cf2cee47ea192cd6f7f4c2e7d904654724c55. * DLPX-80654 [Backport of DLPX-80114 to 6.0.14.0] Add list of installed linux packages for each platform variant during appliance build (#663) (#674) * TOOL-13514 [Backport of TOOL-13469 to 6.0.14.0] appliance-build changes to add telegraf support (#675) * Revert "DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636)" This reverts commit 74543b0c46d2f9a8f85e1cb33dc199d02d447bb5. * DLPX-80424 [Backport of DLPX-80287 to 6.0.15.0] remove unnecessary "recommended" packages from appliance * DLPX-80917 [Backport of DLPX-80873 to 6.0.15.0] Upgrade verification gets stuck when upgrading from 6.0.13.0 -> 6.0.13.1 (#677) * CP-8412 [Backport of CP-8404 to 6.0.15.0] disable default telegraf service (#684) * QA-35796 [Backport of QA-35550 to 6.0.15.0] Add nmap to qa-internal appliance builds (#679) * DLPX-75209 -- Fix for upgrade scenario * DLPX-75209 rebooting a Delphix Engine in AWS can leave the system with no network configuration * Update execute Update the execute script to follow guidelines of shfmt script. * Update execute Incorporate changes suggested by the shellcheck script * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> --- branch.config | 2 +- upgrade/upgrade-scripts/execute | 41 ++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/branch.config b/branch.config index cedb2987..9e572e67 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="master" +UPSTREAM_BRANCH="6.0/stage" diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index ac1192ac..7a3f04f8 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -31,10 +31,21 @@ function usage() { } function generate_interface_to_mac_address_map() { - ip -o link | grep -v "lo:" | tr -d ":" | awk '{print $2,$17}' | \ - tee "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" - [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || \ - die "Failed to generate map from interface names to mac addresses" + # we always create the map file anew to avoid using any stale versions of the same + if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then + rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || + die "Could not delete the existing contents of the mac address map file" + fi + for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do + interface_name="$i" + if [ -e /sys/class/net/"$i"/address ] && + grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then + mac_address=$(tr -d ":" >"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + fi + done + [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || + die "Failed to generate map from interface names to mac addresses" } while getopts ':rlBfsp:' c; do @@ -495,21 +506,13 @@ fi # The container does not have visibility into the complete network namespace of the # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map -if ! systemd-detect-virt --container --quiet; then - platform_type=$(cat /var/lib/delphix-appliance/platform) - if [[ "$platform_type" == "aws" ]]; then - # we always create the map file anew to avoid using any stale versions of the same - if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then - rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || \ - die "Could not delete the existing interface to mac address map file" - fi - # While booting back to the original image post rollback, we want to reuse the original - # netplan file that consisted of the canonical interface names generated by the kernel. - # Hence, we skip creating the map file in rollback execution path - if [[ "$1" != "rollback" ]]; then - generate_interface_to_mac_address_map - fi - fi +if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then + # While booting back to the original image post rollback, we want to reuse the original + # netplan file that consisted of the canonical interface names generated by the kernel. + # Hence, we skip creating the map file in rollback execution path + if [[ "$1" != "rollback" ]]; then + generate_interface_to_mac_address_map + fi fi # From 93e242ba00a8de944525cd20c49a0b896ea14736 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Fri, 2 Sep 2022 08:52:10 +0530 Subject: [PATCH 143/197] Update execute Fix the syntax of find command in function generate_interface_to_mac_address_map --- upgrade/upgrade-scripts/execute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 7a3f04f8..dfe59ca9 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -36,7 +36,7 @@ function generate_interface_to_mac_address_map() { rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi - for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do + for i in $(find /sys/class/net/ -type l | awk -F "/" '{print $5}'); do interface_name="$i" if [ -e /sys/class/net/"$i"/address ] && grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then From 27d68f10d410a4142b17e3a45256c3ab10c0fd42 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Fri, 2 Sep 2022 08:58:49 +0530 Subject: [PATCH 144/197] Update execute Fix syntax of find command in function generate_interface_to_mac_address_map --- upgrade/upgrade-scripts/execute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 7a3f04f8..dfe59ca9 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -36,7 +36,7 @@ function generate_interface_to_mac_address_map() { rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi - for i in $(find /sys/class/net/ type '-l' | awk -F "/" '{print $5}'); do + for i in $(find /sys/class/net/ -type l | awk -F "/" '{print $5}'); do interface_name="$i" if [ -e /sys/class/net/"$i"/address ] && grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then From f6913d2d4f9bfd47f506104ee7efcdf1e23a650a Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Thu, 8 Sep 2022 13:35:38 +0530 Subject: [PATCH 145/197] Merge from DLPX-75209 into 6.0/stage to test the fix for DLPX-75209 on 6.0/stage branch (#2) * 6.0/stage (#1) * [Backport of LB-209 to 6.0/release] Want to consume new mirror for production appliance build * [Backport to 6.0.0.0] Ensure that we are using secondary mirror (#371) * [Backport of LB-209 to 6.0/stage] Want to consume new mirror for production appliance build (#370) * [Backport to 6.0] Ensure that we are using secondary mirror (#372) * [Backport of LB-209 to 6.0/patch] Want to consume new mirror for production appliance build (#376) * [Backport to 6.0] Ensure that we are using secondary mirror (#377) * DLPX-66713 [Backport of Issue DLPX-66369 to 6.0.1.0] Using "upgrade-container stop" may not stop the running upgrade container (#375) * Make debug symbol packages (ddebs) available for installation (#378) * DLPX-66331 [Backport of Issue DLPX-66286 to 6.0.0.0] Interrupting a running upgrade verify can leave the engine not-upgradable (#385) * [Backport to 6.0.0.0] Set 'elevator=noop' GRUB option for first boot after migration (#386) * DLPX-66697 [Backport of Issue DLPX-66696 to 6.0.0.0] migration: add stress options in the pre-reboot dx scripts (#390) * DLPX-67336 [Backport of Issue DLPX-67184 to 6.0.0.0] migration: dx_apply should call dx_delete to cleanup previous state (#389) * DLPX-67355 [Backport of Issue DLPX-66957 to 6.0.0.0] hostname is not locally resolvable (#392) At boot time, applications will fail to resolve the local hostname to an IP address (i.e. the loopback address) until DNS servers are reachable, and once they are reachable, such lookups will result in external DNS requests, which is suboptimal given how often applications resolve the local hostname. These lookups should all be done locally without relying on external DNS servers. When applications resolve a hostname to an IP address, they use the NSS switch, which has a policy for how lookups should be done. On Delphix, the hosts entry for /etc/nsswitch.conf looks like this: hosts: files dns mymachines This means that any lookup will first consult local files (/etc/hosts), and if that fails, will go out to DNS. The /etc/hosts file is managed by cloud-init, and will contain the local hostname in cases where Delphix is running in the cloud. However, when Delphix is not in a cloud environment (e.g. on ESX), cloud-init does not update /etc/hosts with the local hostname (e.g. when it comes from a DHCP option). Lookups then end up going to DNS. The DNS configuration is in /etc/resolv.conf. Today, this file is a symbolic link to /run/systemd/resolve/resolv.conf, which in a DHCP environment, contains the set of DNS servers obtained via DHCP. As such, even requests to resolve the local hostname will end up going out to a remote DNS server. Ubuntu does, however, ship with a local caching DNS server that is built into systemd-resolved, and this caching DNS server has built-in support for resolving the local hostname without having to send such queries externally. Delphix doesn't use this because that would require that /etc/resolv.conf include a "localhost" DNS server so that DNS queries are made to the local DNS server. This can be accomplished by having /etc/resolv.conf be a symbolic link to /run/systemd/resolve/stub-resolv.conf. This resolver configuration points to the localhost server (systemd-resolved), which forwards queries that it wasn't able to satisfy from its local cache to external servers. This also happens to be the recommended default configuration according to the systemd-resolved.service man page. This fix simply changes the default symlink target for /etc/resolv.conf to be /run/systemd/resolve/stub-resolv.conf. * DLPX-67393 [Backport of Issue DLPX-67392 to 6.0.0.0] azure marketplace requires a 1mb unallocated region (#394) * DLPX-67946 [Backport of Issue DLPX-67884 to 6.0.0.0] need Hyper-V image (#403) * Stop using Docker when executing the build (#407) * Disable integration with Jenkins automation * Leverage Github Actions in favor of TravisCI (#398) This change adds the necessary files and logic to take advantage of GitHub Actions for running our automated tests. Currently we rely on TravisCI to do this, but we'd like to transition over to GitHub Actions, for the following reasons: 1. The Actions UI is more integrated with GitHub PRs 2. Best practices at Delphix suggests using Actions Additionally, this change removes integration with the "bors" bot. * Stop using Docker when executing the build (#404) * Modify propagation type of root dataset mount before mounting children (#402) (#408) Co-authored-by: John Gallagher * DLPX-68030 [Backport of Issue DLPX-67937 to 6.0.0.0] need OCI image (#410) * Remove (now) unused Docker related logic (#405) (#411) * [Backport to 6.0.1.0] Disable apt pipelining for the whole duration of live-build * [Backport to 6.0.0.1] Disable apt pipelining for the whole duration of live-build * Remove (now) unused Docker related logic (#405) (#418) * DLPX-67625 [Backport of Issue DLPX-67598 to 6.0.1.0] VDBs becomes inactive during some deferred upgrades from 6.0.1.0, similar to DLPX-66733 (#427) * DLPX-66981 Add verbose logging to the upgrade bash scripts (#397) * Log upgrade scripts' stdout and stderr via syslog (#414) This change builds on the prior commit 9ba7066. Now, in addition to logging the upgrade scripts' execution trace to syslog, we allow scripts to log the stdout and stderr of their commands to syslog as well. This new functionality is leveraged in the "execute" script, such that we keep a better log of what "apt-get" is doing when it's upgrading packages on the system. * Avoid removing "etab" file on upgrade (#421) * DLPX-66509 [Backport of Issue DLPX-66313 to 6.0.1.0] files previously installed into "/etc" by packages may not be removed on upgrade (#428) * Remove obsolete configuration files on upgrade (#425) * [Backport of DLPX-68440 to 6.0.1.0] remove server.conf and demo image from appliance-build (#429) * DLPX-68603 [Backport of DLPX-68595 to 6.0.1.0] delphix-fluentd service disabled during upgrade: fix for 6.0.0.0 (#436) * [Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0 * DLPX-68848 [Backport of DLPX-68832 to 6.0.2.0] DFE during trunk to trunk deferred upgrade, Failed to set-bootfs for container 'delphix.AvZ8Xle'. (#443) * Revert "[Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0" This reverts commit 7868fcdad6b620549fec13aee137c4429d46e92f. * DLPX-68826 [Backport of DLPX-68825 to 6.0.2.0] delphix-fluentd service disabled after not-in-place upgrade * DLPX-68769 [Backport of DLPX-68763 to 6.0.2.0] crashdump quota on rpool gets reset on upgrade (#453) * Backports of 4 commits to add 'upgrade finalize' to 6.0/stage (#457) * [Backport to 6.0.2.0] Add new "finalize" subcommand to upgrade script (#413) This change adds the "finalize" subcommand to the upgrade script, which is intended to be used to clean up any filesystems, snapshots, etc. that may have been generated during the upgrade process (e.g. for rollback). This is intended to be used to mark the finalization or completion of the upgrade, after which rollback will no longer be possible; e.g. $ sudo /var/dlpx-update/latest/upgrade deferred $ sudo /var/dlpx-update/latest/upgrade finalize * [Backport to 6.0.2.0] "rootfs-container delete" must process "log" dataset (#448) When "/var/log" was moved to a seperate dataset in commit 76a37744, we failed to properly adapt the "rootfs-container delete" script to account for this change. Since then, the "rootfs-container delete" script has likely been broken; currently, when I attempt to use this script, it fails with the following error: $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.xfXCbRc cannot destroy 'rpool/ROOT/delphix.xfXCbRc': filesystem has dependent clones use '-R' to destroy the following datasets: rpool/ROOT/delphix.iluoSLh/log rootfs-container: 'zfs destroy -r rpool/ROOT/delphix.xfXCbRc' failed The problem is, we're not properly handling the "log" dataset clones. The fix is simple, we need to add the "log" dataset to the list of datasets that we account for (i.e. the datasets that may need to be "zfs promote"-ed) when calling "rootfs-container delete". * [Backport to 6.0.2.0] Remove lingering snapshots after rollback event (#449) After performing a rollback of a failed not-in-place upgrade, if we remove the container associated with that failure (i.e. the "new" container), we will not remove the "container-delphix.XXXXXXX" snapshots used to generate that new container, and these snapshots will persist indefinitely. This change adds the necessary logic to the "rootfs-container delete" script to remove these snapshots, after removing the container itself. For example, without this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.Bs4VTPo $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU@container-delphix.Bs4VTPo 0B - 64K - rpool/ROOT/delphix.jDHoOtU/data 34.1M 44.3G 33.1M legacy rpool/ROOT/delphix.jDHoOtU/data@container-delphix.Bs4VTPo 1006K - 30.5M - rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/home@container-delphix.Bs4VTPo 151K - 11.8G - rpool/ROOT/delphix.jDHoOtU/log 28.7M 44.3G 27.0M legacy rpool/ROOT/delphix.jDHoOtU/log@container-delphix.Bs4VTPo 1.61M - 3.95M - rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / rpool/ROOT/delphix.jDHoOtU/root@container-delphix.Bs4VTPo 2.02M - 5.86G - And now, with this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.lJvMWgM $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU/data 34.5M 44.3G 34.5M legacy rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/log 27.1M 44.3G 27.1M legacy rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / Thus, with this change, the "container-delphix.XXXXXXX" snapshots are properly removed after calling "rootfs-container delete". * DLPX-69176 [Backport of DLPX-68157 to 6.0.2.0] Update 'finalize' logic to clean up snapshots older than current and previous Co-authored-by: Prakash Surya * Use "delphix-upgrade-verification" package in "verify-jar" script (#455) (#463) DLPX-64198 define and implement version info file for verification package DLPX-68919 Use New upgrade-verify.jar in appliance-build DLPX-69031 Make sure that the delphix-verification package is not installed post upgrade * correct branch.config * copy static DNS configuration for not-in-place upgrades (#470) * Remove unnecessary files from upgrade payload (#472) (#473) Currently, when we generate an upgrade image, we include the entire contents of "~/.aptly" in the image. It turns out, this causes us to include two identical copies of each package, one under "~/.aptly/pool" and another under "~/.aptly/public/pool". For the upgrade image, we only need the contents of the "~/.aptly/public" directory, so this change modifies the upgrade image generation logic, as well as the upgrade image application logic, to take this into account. As a result, this significantly reduces the size of the upgrade image (roughly a 50% reduction in size). * Checkout the master branch instead of projects/dx4linux on dcol. (#475) The projects/dx4linux branch in dcenter-gate is no longer used. All work is being done on master, and so that's the branch that needs to be checked out on deployed dcol instances. * dcenter image should increase mountd thread count [Backport of TOOL-9772] (#477) * backport tenacity requirement (part of TOOL-9418) (#478) * DLPX-71245 [Backport of DLPX-71244 to 6.0.3.0] fix for DLPX-69049 needs to also update grub for migrations (#480) * DLPX-70914 [Backport of DLPX-70835 to 6.0.4.0] disable usb-storage device module loading in delphix 6.x (5.3 migration part) (#483) * DLPX-71311 [Backport of DLPX-69447 to 6.0.4.0] Fixed hotfixes should be removed from /etc/hotfix during upgrade application (#484) * Backport: Install recovery env (#487) * Fix calls to "verify_upgrade_not_in_progress" (#462) (#486) The "verify_upgrade_not_in_progress" function only works correctly if the "UPGRADE_TYPE" variable is not set prior to the function being called. Unfortunately, in the two places we currently call that function, that variable will have already been set. This change fixes the issue by moving the calls to the function, such that we call it prior to setting the variable. Additionally, we add a check to the function to catch this error in usage earlier, and emit a more appropriate error message. Closes #461 * Fix "upgrade-container stop" return code on success (#433) (#434) This change fixes a regression introduced in acb60f4c. Without this change, calling "upgrade-container stop" will return an error code, even when the call successfully stops the container. With this change, when the container is stopped successfully, we'll now properly return 0. * fix branch.config * fix branch.config * Backports of Python package changes (#492) * Add py-six to DCenter appliance-build ansible role (#485) Co-authored-by: brandon.lim * Update DCenter Python package list (#488) - Add python-dbg and python-dev for use with GDB. - Remove python-requests, since we no longer depend on it as of TOOL-9913. - Add python-pyvmomi, because we eventually plan to migrate from PySphere to pyVmomi. Installing the package now enables us to begin preliminary testing with pyVmomi. - Add python-virtualenv, because we eventually plan to deploy our dependencies in a virtual environment. Installing the package now enables us to begin preliminary testing of virtual environments. - Add Python 3 versions of all Python 2 packages, because we eventually plan to migrate from Python 2 to Python 3. Installing Python 3 now enables us to begin preliminary testing with Python 3. Co-authored-by: brandon.lim * TOOL-10595 [Backport of TOOL-10160 to 6.0.6.0] linux-pkg rework: main appliance-build change (#496) * Github Action "::set-env" is no longer supported (#501) (#502) The Github Action "::set-env" command is no longer supported and results in failures when it is used. This change updates our scripts to remove the usage of that command, and use the "GITHUB_ENV" interface instead. Closes #495 * DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install (#499) * DLPX-73083 [Backport of DLPX-72429 to 6.0.6.0] ssh to admin CLI shows error message after force-not-in-place upgrade, permissions issue on /export dir (#504) * DLPX-72721 Cleanup old Illumos datasets on Linux to Linux upgrade (#500) (#505) * Backports of DCenter package updates (#508) Amends #488. Updates the DCenter package list to improve debuggability, adapt to recent changes in dcenter-gate, and prepare for future work. - Adding debug symbols via nfs-common-dbgsym and nfs-kernel-server-dbgsym to improve debuggability. - Removing python3-paramiko, as it is no longer needed when running dcenter-gate with pyVmomi. Note that Paramiko must remain installed for Python 2 until we drop support for PySphere (scheduled for later this month). - Removing virtualenv in favor of venv as in QI-1699. Note that we do not currently use a virtual environment, but we plan to start using one in future packaging and deployment work. We will likely be Python 3 only by the time that work is begun, so it makes more sense to use the native Python 3 venv than virtualenv. - Adding targetcli-fb in order to support prototyping dc volume. Also removes dangling references to dcenter.conf. * Revert "DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install" (#512) This reverts commit c26b447492dd39d87234f1dd2f9f8b49f3786874. * QA-25826 [Backport of QA-20893] Add the snmptrapd pkg to the delphix VM (#513) Co-authored-by: Neetha Menon * Backport hot-add to 6.0.6.0 (#516) * [Backport to 6.0.6.0] Enable generation of "sparse" upgrade image (#514) This change adds a new script that can be used to generate a "sparse" upgrade image. For example, given two input upgrade images, we can generate a sparse image like so: $ ./scripts/aptly-repo-from-image-diff.sh ../master/internal-dev.upgrade.tar ../6.0-release/internal-dev.upgrade.tar $ ./scripts/upgrade-image-from-aptly-repo.sh sparse-internal-dev $ ls -l build/artifacts/sparse-internal-dev.upgrade.tar -rw-r--r-- 1 delphix staff 5459210240 Jul 31 19:47 build/artifacts/sparse-internal-dev.upgrade.tar Additionally, we can inspect this new artifact and verify the set of packages it contains: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 199 And contrast this with the number of packages contained in both the "master" and "6.0-release" images: - master: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1042 - 6.0/release: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1080 Co-authored-by: Prakash Surya * DLPX-73378 [Backport of DLPX-73299 to 6.0.6.0] appliance-build: Fetch upgrade-verify package from combined-packages bundle (#515) * TOOL-10893 High disk space usage for audit logs threatens the success of future upgrades (#518) (#519) * CP-4470 [Backport of CP-4449 to 6.0.7.0] use pvscsi by default (#522) CP-4471 [Backport of CP-4451 to 6.0.7.0] Bump OVA hardware version to HWv11 * Backporting #523, #524, #525, and #526. (#527) * Backporting #523, #524, #525, and #526. (#528) * Fix branch.config * DLPX-71882 [Backport of DLPX-71833 to 6.0/stage] ui-precommit fails to start ChromeHeadless (#530) * Initial changes to support new hotfix workflow(s) (#507) (#531) * Support "hotfix-version" property on build and upgrade * Support hotfix version in upgrade image * Set hotfix version via "execute" script Co-authored-by: Prakash Surya * DLPX-75090 [Backport of DLPX-75089 to 6.0.8.0] Upgrade verification should not enable nfs services (#535) * DLPX-75232 [Backport of DLPX-75229 to 6.0.8.0] finalize() should not rely on properties in upgrade.properties (#539) * DLPX-75258 [Backport of DLPX-73923 to 6.0.9.0] Persist iSCSI initiator files during not-in-place upgrade (#542) * DLPX-75450 [Backport of DLPX-75347 to 6.0.9.0] old rootfs versions are not removed on upgrade in 6.0.8.0 (#545) * DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547) * DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549) * Don't remove "delphix-entire" when generating image diffs (#536) (#537) Currently, if we generate an upgrade image diff from two images that have the same delphix-entire package version, the resultant Aptly repository will not contain any delphix-entire package in it. This is problematic when that Aptly repository is later used with the script used to generate an upgrade image from that repository, as the script requires the repository to contain a "delphix-entire" package in it. Thus, to avoid this issue, this change modifies the image diff logic to ensure we never remove the "delphix-entire" package. * DLPX-75091 DLPX-75122 Make upgrade image prepare script hotfix aware (#543) (#550) DLPX-75091 Make upgrade image prepare script hotfix aware DLPX-75122 Use delphix version and hotfix in version.info in verify-jar script * Revert "DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549)" (#555) This reverts commit f6421aa1f2112ec17eb53737d079ff87f1e1a533. * Revert "DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547)" (#556) This reverts commit 26c960f1697c1cef7d66bae8326670094b9e206d. * DLPX-76243 management stack failed to come back after rollback upgrade (#558) * CP-5356 [Backport of CP-5334 to 6.0.9.0] Add ufw package to appliance-build (#559) * DLPX-76329 [Backport of DLPX-76328 to 6.0.9.0] upgrade script obtains hotfix version incorrectly (#564) In commit e7dd85e2 we introduced the `get_hotfix_version` function for obtaining the hotfix version of the currently running root filesystem. The problem is that in c4357a4e we added logic that intended to use this function, but instead tries to call the `get_current_hotfix` function, which doesn't exist. This leads us to never retreiving the hotfix version correctly via the `rollback` subcommand of the `upgrade` script. * DLPX-74045 adapt upgrade "execute" script to better handle new hotfix workflow (#520) (#568) Currently when applying a hotfix image generated by the new hotfix workflow being developed in CP-4190, nothing will happen when the upgrade is attempted. This is due to the "delphix-entire" package in the hotfix being installed, matching the version of the package that's already installed. Thus, the package manager chooses not to install the new packagecontained in the upgrade/hotfix image, since it thinks the same package is already installed, due to the two having the same version. We need to adapt the "execute" script that's used during upgrade, such that we always install the new "delphix-entire" package contained in the upgrade/hotfix image, even if the package versions are identical. * TOOL-11824 Harden against mistakes when running via Jenkins (#569) (#570) This change adds logic to cause a failure when certain environment variables are missing, and the build is being run via Jenkins. When environment variables are missing, we currently will provide default values, such that the build doesn't fail. These default values generally are not sufficient for anything besides allowing the build to complete (e.g. the resultant upgrade images may fail upgrade). Further, the main motivation for generating these default values, is to make it easier to run the build manually, and aid developer iteration. Thus, when it's clear that the build is being run via Jenkins, where these missing environment variables more likely point to an error or broken automation, than an intentional omission of these variables, it's better to report the error than set and use a default value. This way, we can more quickly detect and address the problem, than silently produce (potentially) incorrect or broken artifacts. * [Backport to 6.0] Remove code for building migration images (#571) Co-authored-by: John Gallagher * DLPX-76433 [Backport of DLPX-76376 to 6.0.10.0] rootfs property "com.delphix:hotfix-version" incorrect after rollback of hotfix (#573) * DLPX-76478 [Backport of DLPX-71683 to 6.0.10.0] Generate new buildinfo package using metadata from combined packages (#493) (#572) * Add rust to dev environment (#544) (#577) Co-authored-by: Paul Dagnelie * Backport a few changes from master to 6.0/stage (#579) * remove crash-python role from appliance-build Now that we can run crash-python outside the git repo from any path and we are halting future development on it, its role on the appliance build is no longer needed for developer VMs. Note that deleting that role uncovered a different bug where the minimal-common role in internal-dev is needed by the ldap role and it would always run before it because it was listed as a dependency under the crash python role. Now that the crash-python role is removed we re-add an old version of that configuration making the minimal-common role run before the ldap role. * Add "shell" subcommand to upgrade-container script (#420) This adds a new "shell" subcommand to the "upgrade-container" script, which is intended to enable users to obtain an interactive shell session within the specified container. Currently, the "run" subcommand can be used, while specifying the shell to run (e.g. /bin/bash), but since that does not specify the "--pty" option with "systemd-run", the terminal may not work correctly (e.g. keyboard input, pager output, etc). Now, when folks want to obtain an interactive shell, they should use this new "shell" subcommand, rather than "run". * Fix "upgrade-container shell" argument handling (#426) Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Prakash Surya * DLPX-76770 [Backport of DLPX-68846] logic for determining device path for grub operations during upgrade is fragile (#581) * DLPX-76769 [Backport of DLPX-76694 to 6.0.10.0] Failure to build DelphixConnector on Delphix Engine, causing gui_sanity testing to fail (#580) * DLPX-76293 Copy Hotfix Metadata Generation to appliance-build (#574) (#590) Co-authored-by: Matt Skinner * install-ansible-lint is failing (#587) (#593) Co-authored-by: Pavel Zakharov * DLPX-76893 [Backport of DLPX-76802 to 6.0.10.0] Starting Verification container enables IP forwarding on host (#592) * Want variant-specific package lists for `live-build(7)` (#596) * DLPX-77250 [Backport of DLPX-74852 to 6.0.11.0] Upgrade subsystem leaves behind stale entry in /etc/apt/sources.list (#602) * TOOL-12103 [Backport of TOOL-12101 to 6.0/stage] ui-precommit fails with missing necessary HEADLESS chrome dependency (#604) * Use "delphix-rust" instead of Ubuntu's Rust packages (#606) * CP-5971 [Backport of CP-5946 to 6.0.11.0] Enable Java debug mode for sso app (#610) * DLPX-77019 [Backport of DLPX-76998 to 6.0.11.0] CLONE - Starting Verification container enables IP forwarding on host (#611) * DLPX-76940 [Backport of DLPX-76907 to 6.0.11.0] obsolete conf file cleanup logic incorrectly removes conf files that were moved to another package (#613) * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12005 Enable ntp on dcenter-internal variant (#594) * Use HTTPS URL for Artifactory (#598) * DLPX-77385 Upgrade: all packages should be upgraded at the same time, remove exception for delphix-platform (#608) * TOOL-12324 appliance-build: add a retry when unmount fails with EBUSY (#617) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab (#619) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab * fixes * DLPX-77878 Fix debootstrap arguments when creating not-in-place upgrade container (#616) * TOOL-12323 Remove libnss3-dbg package as it is missing on Ubuntu 20.04 (#618) * QI-3191 dlpx-qa-gate not installing on DCoLs as Jenkins agents (#623) Co-authored-by: Brandon Lim * TOOL-12368 appliance-build: update some python2 package dependencies to python3 (#621) * DLPX-78255 ui-precommit fails due to missing Chrome dependency libnss3.so (#624) * DLPX-78304 [Backport of DLPX-77901 to 6.0.12.0] Prevent services from being re-enabled on upgrade (#627) * DLPX-78328 [Backport of DLPX-78275 to 6.0.12.0] Upgrade-verification is failing because /sys/module is not available in container (#629) * DLPX-78436 [Backport of DLPX-78371 to 6.0.12.0] Upgrade fails because some packages are unexpectedly autoremoved by apt (#635) Co-authored-by: Pavel Zakharov * DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636) * TOOL-12541 [Backport of TOOL-12471 to 6.0.12.0] Remove Jenkins job references to devops-gate/master in appliance-build (#637) * Don't use "apt-get autoremove" anymore (#642) (#643) We've discovered that "apt-get autoremove" does not work for our needs. Specifically, packages may erroneously remain installed due to suggested and recommended package dependencies, as well as due to virtual package dependencies. While we could alleviate the issue w.r.t. suggested and recommended dependencies via configuration changes, there isn't a clear way to resolve the issue for virtual packages. Thus, the solution adopted in this change is to simply not use "apt-get automoremove" anymore, and instead, explicitly "apt-get purge" all package that we intend to be removed. This addresses the following two bugs: * DLPX-64201 determine how to remove old linux kernels * DLPX-78220 Old packages not removed after deferred upgrade * DLPX-78307 Upgrade tests to trunk failed (#647) (#649) * DLPX-77974 Add minimum version property to root dataset (#620) (#650) * TOOL-12418 [Backport of TOOL-12369 to 6.0.13.0] appliance-build: changes for Ubuntu 20.04 (#653) * CP-7160 [Backport of CP-6676 to 6.0.13.0] appliance-build changes to simplify the fluentd build (#655) * DLPX-79489 [Backport of DLPX-79367] td-agent service is not masked after upgrade (#659) * DLPX-79276 [Backport of DLPX-78736 to 6.0.13.0] Several test_api_and_cli_list_operation tests failed after stage to trunk upgrade (#660) * Rely on pre-installed "awscli" package (#661) (#662) * DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670) * DLPX-79030 [Backport of DLPX-78464 to 6.0.14.0] Regression in appliance-build from DLPX-76293 (#669) * Revert "DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670)" (#672) This reverts commit 0d0cf2cee47ea192cd6f7f4c2e7d904654724c55. * DLPX-80654 [Backport of DLPX-80114 to 6.0.14.0] Add list of installed linux packages for each platform variant during appliance build (#663) (#674) * TOOL-13514 [Backport of TOOL-13469 to 6.0.14.0] appliance-build changes to add telegraf support (#675) * Revert "DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636)" This reverts commit 74543b0c46d2f9a8f85e1cb33dc199d02d447bb5. * DLPX-80424 [Backport of DLPX-80287 to 6.0.15.0] remove unnecessary "recommended" packages from appliance * DLPX-80917 [Backport of DLPX-80873 to 6.0.15.0] Upgrade verification gets stuck when upgrading from 6.0.13.0 -> 6.0.13.1 (#677) * CP-8412 [Backport of CP-8404 to 6.0.15.0] disable default telegraf service (#684) * QA-35796 [Backport of QA-35550 to 6.0.15.0] Add nmap to qa-internal appliance builds (#679) * DLPX-75209 -- Fix for upgrade scenario * DLPX-75209 rebooting a Delphix Engine in AWS can leave the system with no network configuration * Update execute Update the execute script to follow guidelines of shfmt script. * Update execute Incorporate changes suggested by the shellcheck script * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> * Update execute Fix syntax of find command in function generate_interface_to_mac_address_map Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> From 3d22293f40eb74929a7fd6b659482786db486936 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Sun, 11 Sep 2022 22:25:00 +0530 Subject: [PATCH 146/197] Update execute Simplify the logic in generate_interface_to_mac_address_map method through the following changes -- 1. Allow an entry for the loopback interface in the interface <-> mac address map file. This presence of the loopback interface in the map file is harmless and simplifies the logic. 2. Skip the truncation of ":" characters in the mac_address. This truncation will executed in the script /usr/bin/update_netplan_to_mac_address which is managed by the delphix-platform repository and updates the netplan file to mac address based interface names while booting into the upgraded image post engine upgrade. --- upgrade/upgrade-scripts/execute | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index dfe59ca9..68af6cc9 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -36,12 +36,11 @@ function generate_interface_to_mac_address_map() { rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi - for i in $(find /sys/class/net/ -type l | awk -F "/" '{print $5}'); do - interface_name="$i" - if [ -e /sys/class/net/"$i"/address ] && - grep -q -v -e "00:00:00:00:00:00" /sys/class/net/"$i"/address; then - mac_address=$(tr -d ":" >"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" + for i in /sys/class/net/* ; do + ifname=$(basename "$i") + if [ -e "$i"/address ]; then + mac_address=$(cat "$i"/address) + echo "$ifname" "$mac_address" >>"$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" fi done [ -s "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ] || From 5fded00b15a06410cc78f621591bb575cd72ea25 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Sun, 11 Sep 2022 22:48:12 +0530 Subject: [PATCH 147/197] Update execute Incorporate changes suggested by the shfmt script. --- upgrade/upgrade-scripts/execute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 68af6cc9..d4b9b04c 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -36,7 +36,7 @@ function generate_interface_to_mac_address_map() { rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || die "Could not delete the existing contents of the mac address map file" fi - for i in /sys/class/net/* ; do + for i in /sys/class/net/*; do ifname=$(basename "$i") if [ -e "$i"/address ]; then mac_address=$(cat "$i"/address) From 18810d5816ba5c2283a190ea331608cd996cd895 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:11:20 +0530 Subject: [PATCH 148/197] Update execute Incorporate changes suggested in the review. 1. Removed the check for "rollback" type before calling the "generate_interface_to_mac_address_map" function. 2. Changed the target location for the map file from /etc/ to /var/tmp/delphix-upgrade --- upgrade/upgrade-scripts/execute | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index d4b9b04c..ebe73317 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -17,7 +17,7 @@ . "${BASH_SOURCE%/*}/common.sh" -INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" +INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/var/tmp/delphix-upgrade/interface_to_macaddress_map.out" IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" @@ -484,9 +484,9 @@ fi # entries in the netplan plan, because they are not to be found on the new instance. # However, the mac address remains consistent and does not change with a change in # the instance types. -# To handle this, from 6.0.16.0 onwards, we are switching to use mac address based +# To handle this, from 6.0.17.0 onwards, we are switching to use mac address based # interface names in the netplan file. To facilitate this change across upgrades from -# older verrsions to 6.0.16.0 and above, we create a map from the network interface +# older verrsions to 6.0.17.0 and above, we create a map from the network interface # names to their corresponding mac address, resulting in mac addrress based interface # names. For wider context, see DLPX-75209. # @@ -496,10 +496,10 @@ fi # map file from the currently mounted root filesystem to the root filesystem of the # target container. # -# The map creation logic does not apply to engines being upgraded from 6.0.16.0 onwards. -# This is because the netplan file from 6.0.16.0 will by default be populated with mac +# The map creation logic does not apply to engines being upgraded from 6.0.17.0 onwards. +# This is because the netplan file from 6.0.17.0 will by default be populated with mac # address based interface names. This makes this logic redundant for upgrades from -# 6.0.16.0 and onwards. Hence, we must remove this logic as soon as we mark 6.0.16.0 as +# 6.0.17.0 and onwards. Hence, we must remove this logic as soon as we mark 6.0.17.0 as # the minimum version supported release. # The container does not have visibility into the complete network namespace of the From 7b76ded16ed1eddbe536c50898a77b5a468423f3 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:13:42 +0530 Subject: [PATCH 149/197] Update execute Removed the check for "rollback" type before calling the generate_interface_to_mac_address_map function. --- upgrade/upgrade-scripts/execute | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index ebe73317..2a402e1f 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -506,12 +506,7 @@ fi # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then - # While booting back to the original image post rollback, we want to reuse the original - # netplan file that consisted of the canonical interface names generated by the kernel. - # Hence, we skip creating the map file in rollback execution path - if [[ "$1" != "rollback" ]]; then - generate_interface_to_mac_address_map - fi + generate_interface_to_mac_address_map fi # From f335fd6226cb8023ddff11e7c4f7bfbd307acd37 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Tue, 13 Sep 2022 15:53:45 +0530 Subject: [PATCH 150/197] Update execute Moved the definition of INTERFACE_TO_MACADRESS_MAP_FILE_PATH variable inside the definition of generate_interface_to_mac_address_map() function. --- upgrade/upgrade-scripts/execute | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 2a402e1f..8bb02b0a 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -17,8 +17,6 @@ . "${BASH_SOURCE%/*}/common.sh" -INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/var/tmp/delphix-upgrade/interface_to_macaddress_map.out" - IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" @@ -31,6 +29,7 @@ function usage() { } function generate_interface_to_mac_address_map() { + INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/var/tmp/delphix-upgrade/interface_to_macaddress_map.out" # we always create the map file anew to avoid using any stale versions of the same if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || From fb290a0ac56fc3d4a2a2846bb4ad3bfe246d5ac5 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Fri, 16 Sep 2022 22:21:41 +0530 Subject: [PATCH 151/197] Update execute Modify the location of the interface to mac address map file from /var/tmp/delphix-upgrade to /etc. This is being done because we found out in testing that "/var/tmp/delphix-mgmt/interface_to_macaddress_map.out" is not available (likely does not exist) when the systemd-networkd daemon is started in the initramfs phase while booting up into the new image post upgrade. The "/etc/interface_to_macaddress_map.out" path is available. It seems this is a result of only specific paths getting built into the initramfs image. --- upgrade/upgrade-scripts/execute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 8bb02b0a..b6f73d29 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -29,7 +29,7 @@ function usage() { } function generate_interface_to_mac_address_map() { - INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/var/tmp/delphix-upgrade/interface_to_macaddress_map.out" + INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" # we always create the map file anew to avoid using any stale versions of the same if [[ -e "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" ]]; then rm "$INTERFACE_TO_MACADRESS_MAP_FILE_PATH" || From f1a2d571403b6b07773ac0ab582ddfc4581b0be9 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Sat, 17 Sep 2022 10:46:03 +0530 Subject: [PATCH 152/197] Update execute Remove the global var INTERFACE_TO_MACADRESS_MAP_FILE_PATH. The scope of this var is now local to function generate_interface_to_mac_address_map. --- upgrade/upgrade-scripts/execute | 2 -- 1 file changed, 2 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index c690b4c8..0f8f683d 100644 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -17,8 +17,6 @@ . "${BASH_SOURCE%/*}/common.sh" -INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" - IMAGE_PATH=$(get_image_path) [[ -n "$IMAGE_PATH" ]] || die "failed to determine image path" From 05e7009f4ccb513fe5e9566e4bbc0a04ad61d670 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Singh <91129608+naveen-singh-delphix@users.noreply.github.com> Date: Mon, 19 Sep 2022 08:13:03 +0530 Subject: [PATCH 153/197] Merge from 6.0/stage to DLPX-75209 to fix the permissions on the execute script. The script should have the permission 0755 instead of 0644 (#5) * Merge from DLPX-75209 into 6.0/stage to test the fix for DLPX-75209 on 6.0/stage branch (#3) * 6.0/stage (#1) * [Backport of LB-209 to 6.0/release] Want to consume new mirror for production appliance build * [Backport to 6.0.0.0] Ensure that we are using secondary mirror (#371) * [Backport of LB-209 to 6.0/stage] Want to consume new mirror for production appliance build (#370) * [Backport to 6.0] Ensure that we are using secondary mirror (#372) * [Backport of LB-209 to 6.0/patch] Want to consume new mirror for production appliance build (#376) * [Backport to 6.0] Ensure that we are using secondary mirror (#377) * DLPX-66713 [Backport of Issue DLPX-66369 to 6.0.1.0] Using "upgrade-container stop" may not stop the running upgrade container (#375) * Make debug symbol packages (ddebs) available for installation (#378) * DLPX-66331 [Backport of Issue DLPX-66286 to 6.0.0.0] Interrupting a running upgrade verify can leave the engine not-upgradable (#385) * [Backport to 6.0.0.0] Set 'elevator=noop' GRUB option for first boot after migration (#386) * DLPX-66697 [Backport of Issue DLPX-66696 to 6.0.0.0] migration: add stress options in the pre-reboot dx scripts (#390) * DLPX-67336 [Backport of Issue DLPX-67184 to 6.0.0.0] migration: dx_apply should call dx_delete to cleanup previous state (#389) * DLPX-67355 [Backport of Issue DLPX-66957 to 6.0.0.0] hostname is not locally resolvable (#392) At boot time, applications will fail to resolve the local hostname to an IP address (i.e. the loopback address) until DNS servers are reachable, and once they are reachable, such lookups will result in external DNS requests, which is suboptimal given how often applications resolve the local hostname. These lookups should all be done locally without relying on external DNS servers. When applications resolve a hostname to an IP address, they use the NSS switch, which has a policy for how lookups should be done. On Delphix, the hosts entry for /etc/nsswitch.conf looks like this: hosts: files dns mymachines This means that any lookup will first consult local files (/etc/hosts), and if that fails, will go out to DNS. The /etc/hosts file is managed by cloud-init, and will contain the local hostname in cases where Delphix is running in the cloud. However, when Delphix is not in a cloud environment (e.g. on ESX), cloud-init does not update /etc/hosts with the local hostname (e.g. when it comes from a DHCP option). Lookups then end up going to DNS. The DNS configuration is in /etc/resolv.conf. Today, this file is a symbolic link to /run/systemd/resolve/resolv.conf, which in a DHCP environment, contains the set of DNS servers obtained via DHCP. As such, even requests to resolve the local hostname will end up going out to a remote DNS server. Ubuntu does, however, ship with a local caching DNS server that is built into systemd-resolved, and this caching DNS server has built-in support for resolving the local hostname without having to send such queries externally. Delphix doesn't use this because that would require that /etc/resolv.conf include a "localhost" DNS server so that DNS queries are made to the local DNS server. This can be accomplished by having /etc/resolv.conf be a symbolic link to /run/systemd/resolve/stub-resolv.conf. This resolver configuration points to the localhost server (systemd-resolved), which forwards queries that it wasn't able to satisfy from its local cache to external servers. This also happens to be the recommended default configuration according to the systemd-resolved.service man page. This fix simply changes the default symlink target for /etc/resolv.conf to be /run/systemd/resolve/stub-resolv.conf. * DLPX-67393 [Backport of Issue DLPX-67392 to 6.0.0.0] azure marketplace requires a 1mb unallocated region (#394) * DLPX-67946 [Backport of Issue DLPX-67884 to 6.0.0.0] need Hyper-V image (#403) * Stop using Docker when executing the build (#407) * Disable integration with Jenkins automation * Leverage Github Actions in favor of TravisCI (#398) This change adds the necessary files and logic to take advantage of GitHub Actions for running our automated tests. Currently we rely on TravisCI to do this, but we'd like to transition over to GitHub Actions, for the following reasons: 1. The Actions UI is more integrated with GitHub PRs 2. Best practices at Delphix suggests using Actions Additionally, this change removes integration with the "bors" bot. * Stop using Docker when executing the build (#404) * Modify propagation type of root dataset mount before mounting children (#402) (#408) Co-authored-by: John Gallagher * DLPX-68030 [Backport of Issue DLPX-67937 to 6.0.0.0] need OCI image (#410) * Remove (now) unused Docker related logic (#405) (#411) * [Backport to 6.0.1.0] Disable apt pipelining for the whole duration of live-build * [Backport to 6.0.0.1] Disable apt pipelining for the whole duration of live-build * Remove (now) unused Docker related logic (#405) (#418) * DLPX-67625 [Backport of Issue DLPX-67598 to 6.0.1.0] VDBs becomes inactive during some deferred upgrades from 6.0.1.0, similar to DLPX-66733 (#427) * DLPX-66981 Add verbose logging to the upgrade bash scripts (#397) * Log upgrade scripts' stdout and stderr via syslog (#414) This change builds on the prior commit 9ba7066. Now, in addition to logging the upgrade scripts' execution trace to syslog, we allow scripts to log the stdout and stderr of their commands to syslog as well. This new functionality is leveraged in the "execute" script, such that we keep a better log of what "apt-get" is doing when it's upgrading packages on the system. * Avoid removing "etab" file on upgrade (#421) * DLPX-66509 [Backport of Issue DLPX-66313 to 6.0.1.0] files previously installed into "/etc" by packages may not be removed on upgrade (#428) * Remove obsolete configuration files on upgrade (#425) * [Backport of DLPX-68440 to 6.0.1.0] remove server.conf and demo image from appliance-build (#429) * DLPX-68603 [Backport of DLPX-68595 to 6.0.1.0] delphix-fluentd service disabled during upgrade: fix for 6.0.0.0 (#436) * [Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0 * DLPX-68848 [Backport of DLPX-68832 to 6.0.2.0] DFE during trunk to trunk deferred upgrade, Failed to set-bootfs for container 'delphix.AvZ8Xle'. (#443) * Revert "[Backport of DLPX-68628 to 6.0.1.0] Bump minimum version for migration to 5.3.7.0" This reverts commit 7868fcdad6b620549fec13aee137c4429d46e92f. * DLPX-68826 [Backport of DLPX-68825 to 6.0.2.0] delphix-fluentd service disabled after not-in-place upgrade * DLPX-68769 [Backport of DLPX-68763 to 6.0.2.0] crashdump quota on rpool gets reset on upgrade (#453) * Backports of 4 commits to add 'upgrade finalize' to 6.0/stage (#457) * [Backport to 6.0.2.0] Add new "finalize" subcommand to upgrade script (#413) This change adds the "finalize" subcommand to the upgrade script, which is intended to be used to clean up any filesystems, snapshots, etc. that may have been generated during the upgrade process (e.g. for rollback). This is intended to be used to mark the finalization or completion of the upgrade, after which rollback will no longer be possible; e.g. $ sudo /var/dlpx-update/latest/upgrade deferred $ sudo /var/dlpx-update/latest/upgrade finalize * [Backport to 6.0.2.0] "rootfs-container delete" must process "log" dataset (#448) When "/var/log" was moved to a seperate dataset in commit 76a37744, we failed to properly adapt the "rootfs-container delete" script to account for this change. Since then, the "rootfs-container delete" script has likely been broken; currently, when I attempt to use this script, it fails with the following error: $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.xfXCbRc cannot destroy 'rpool/ROOT/delphix.xfXCbRc': filesystem has dependent clones use '-R' to destroy the following datasets: rpool/ROOT/delphix.iluoSLh/log rootfs-container: 'zfs destroy -r rpool/ROOT/delphix.xfXCbRc' failed The problem is, we're not properly handling the "log" dataset clones. The fix is simple, we need to add the "log" dataset to the list of datasets that we account for (i.e. the datasets that may need to be "zfs promote"-ed) when calling "rootfs-container delete". * [Backport to 6.0.2.0] Remove lingering snapshots after rollback event (#449) After performing a rollback of a failed not-in-place upgrade, if we remove the container associated with that failure (i.e. the "new" container), we will not remove the "container-delphix.XXXXXXX" snapshots used to generate that new container, and these snapshots will persist indefinitely. This change adds the necessary logic to the "rootfs-container delete" script to remove these snapshots, after removing the container itself. For example, without this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.Bs4VTPo $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU@container-delphix.Bs4VTPo 0B - 64K - rpool/ROOT/delphix.jDHoOtU/data 34.1M 44.3G 33.1M legacy rpool/ROOT/delphix.jDHoOtU/data@container-delphix.Bs4VTPo 1006K - 30.5M - rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/home@container-delphix.Bs4VTPo 151K - 11.8G - rpool/ROOT/delphix.jDHoOtU/log 28.7M 44.3G 27.0M legacy rpool/ROOT/delphix.jDHoOtU/log@container-delphix.Bs4VTPo 1.61M - 3.95M - rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / rpool/ROOT/delphix.jDHoOtU/root@container-delphix.Bs4VTPo 2.02M - 5.86G - And now, with this change: $ download-latest-image internal-dev $ sudo unpack-image -x internal-dev.upgrade.tar $ sudo /var/dlpx-update/latest/upgrade -v full $ sudo /var/dlpx-update/latest/upgrade rollback $ sudo /var/dlpx-update/latest/rootfs-container delete delphix.lJvMWgM $ sudo zfs list -t all -r rpool/ROOT/delphix.jDHoOtU NAME USED AVAIL REFER MOUNTPOINT rpool/ROOT/delphix.jDHoOtU 17.7G 44.3G 64K none rpool/ROOT/delphix.jDHoOtU/data 34.5M 44.3G 34.5M legacy rpool/ROOT/delphix.jDHoOtU/home 11.8G 44.3G 11.8G legacy rpool/ROOT/delphix.jDHoOtU/log 27.1M 44.3G 27.1M legacy rpool/ROOT/delphix.jDHoOtU/root 5.86G 44.3G 5.86G / Thus, with this change, the "container-delphix.XXXXXXX" snapshots are properly removed after calling "rootfs-container delete". * DLPX-69176 [Backport of DLPX-68157 to 6.0.2.0] Update 'finalize' logic to clean up snapshots older than current and previous Co-authored-by: Prakash Surya * Use "delphix-upgrade-verification" package in "verify-jar" script (#455) (#463) DLPX-64198 define and implement version info file for verification package DLPX-68919 Use New upgrade-verify.jar in appliance-build DLPX-69031 Make sure that the delphix-verification package is not installed post upgrade * correct branch.config * copy static DNS configuration for not-in-place upgrades (#470) * Remove unnecessary files from upgrade payload (#472) (#473) Currently, when we generate an upgrade image, we include the entire contents of "~/.aptly" in the image. It turns out, this causes us to include two identical copies of each package, one under "~/.aptly/pool" and another under "~/.aptly/public/pool". For the upgrade image, we only need the contents of the "~/.aptly/public" directory, so this change modifies the upgrade image generation logic, as well as the upgrade image application logic, to take this into account. As a result, this significantly reduces the size of the upgrade image (roughly a 50% reduction in size). * Checkout the master branch instead of projects/dx4linux on dcol. (#475) The projects/dx4linux branch in dcenter-gate is no longer used. All work is being done on master, and so that's the branch that needs to be checked out on deployed dcol instances. * dcenter image should increase mountd thread count [Backport of TOOL-9772] (#477) * backport tenacity requirement (part of TOOL-9418) (#478) * DLPX-71245 [Backport of DLPX-71244 to 6.0.3.0] fix for DLPX-69049 needs to also update grub for migrations (#480) * DLPX-70914 [Backport of DLPX-70835 to 6.0.4.0] disable usb-storage device module loading in delphix 6.x (5.3 migration part) (#483) * DLPX-71311 [Backport of DLPX-69447 to 6.0.4.0] Fixed hotfixes should be removed from /etc/hotfix during upgrade application (#484) * Backport: Install recovery env (#487) * Fix calls to "verify_upgrade_not_in_progress" (#462) (#486) The "verify_upgrade_not_in_progress" function only works correctly if the "UPGRADE_TYPE" variable is not set prior to the function being called. Unfortunately, in the two places we currently call that function, that variable will have already been set. This change fixes the issue by moving the calls to the function, such that we call it prior to setting the variable. Additionally, we add a check to the function to catch this error in usage earlier, and emit a more appropriate error message. Closes #461 * Fix "upgrade-container stop" return code on success (#433) (#434) This change fixes a regression introduced in acb60f4c. Without this change, calling "upgrade-container stop" will return an error code, even when the call successfully stops the container. With this change, when the container is stopped successfully, we'll now properly return 0. * fix branch.config * fix branch.config * Backports of Python package changes (#492) * Add py-six to DCenter appliance-build ansible role (#485) Co-authored-by: brandon.lim * Update DCenter Python package list (#488) - Add python-dbg and python-dev for use with GDB. - Remove python-requests, since we no longer depend on it as of TOOL-9913. - Add python-pyvmomi, because we eventually plan to migrate from PySphere to pyVmomi. Installing the package now enables us to begin preliminary testing with pyVmomi. - Add python-virtualenv, because we eventually plan to deploy our dependencies in a virtual environment. Installing the package now enables us to begin preliminary testing of virtual environments. - Add Python 3 versions of all Python 2 packages, because we eventually plan to migrate from Python 2 to Python 3. Installing Python 3 now enables us to begin preliminary testing with Python 3. Co-authored-by: brandon.lim * TOOL-10595 [Backport of TOOL-10160 to 6.0.6.0] linux-pkg rework: main appliance-build change (#496) * Github Action "::set-env" is no longer supported (#501) (#502) The Github Action "::set-env" command is no longer supported and results in failures when it is used. This change updates our scripts to remove the usage of that command, and use the "GITHUB_ENV" interface instead. Closes #495 * DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install (#499) * DLPX-73083 [Backport of DLPX-72429 to 6.0.6.0] ssh to admin CLI shows error message after force-not-in-place upgrade, permissions issue on /export dir (#504) * DLPX-72721 Cleanup old Illumos datasets on Linux to Linux upgrade (#500) (#505) * Backports of DCenter package updates (#508) Amends #488. Updates the DCenter package list to improve debuggability, adapt to recent changes in dcenter-gate, and prepare for future work. - Adding debug symbols via nfs-common-dbgsym and nfs-kernel-server-dbgsym to improve debuggability. - Removing python3-paramiko, as it is no longer needed when running dcenter-gate with pyVmomi. Note that Paramiko must remain installed for Python 2 until we drop support for PySphere (scheduled for later this month). - Removing virtualenv in favor of venv as in QI-1699. Note that we do not currently use a virtual environment, but we plan to start using one in future packaging and deployment work. We will likely be Python 3 only by the time that work is begun, so it makes more sense to use the native Python 3 venv than virtualenv. - Adding targetcli-fb in order to support prototyping dc volume. Also removes dangling references to dcenter.conf. * Revert "DLPX-73044 [Backport of DLPX-72860 to 6.0.6.0] Patch VMDK generation tool to mark that the vmdk contains a vmtools install" (#512) This reverts commit c26b447492dd39d87234f1dd2f9f8b49f3786874. * QA-25826 [Backport of QA-20893] Add the snmptrapd pkg to the delphix VM (#513) Co-authored-by: Neetha Menon * Backport hot-add to 6.0.6.0 (#516) * [Backport to 6.0.6.0] Enable generation of "sparse" upgrade image (#514) This change adds a new script that can be used to generate a "sparse" upgrade image. For example, given two input upgrade images, we can generate a sparse image like so: $ ./scripts/aptly-repo-from-image-diff.sh ../master/internal-dev.upgrade.tar ../6.0-release/internal-dev.upgrade.tar $ ./scripts/upgrade-image-from-aptly-repo.sh sparse-internal-dev $ ls -l build/artifacts/sparse-internal-dev.upgrade.tar -rw-r--r-- 1 delphix staff 5459210240 Jul 31 19:47 build/artifacts/sparse-internal-dev.upgrade.tar Additionally, we can inspect this new artifact and verify the set of packages it contains: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 199 And contrast this with the number of packages contained in both the "master" and "6.0-release" images: - master: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1042 - 6.0/release: $ tar -xf sparse-internal-dev.upgrade.tar $ tar -xf payload.tar.gz $ find pool -type f -name '*.deb' | wc -l 1080 Co-authored-by: Prakash Surya * DLPX-73378 [Backport of DLPX-73299 to 6.0.6.0] appliance-build: Fetch upgrade-verify package from combined-packages bundle (#515) * TOOL-10893 High disk space usage for audit logs threatens the success of future upgrades (#518) (#519) * CP-4470 [Backport of CP-4449 to 6.0.7.0] use pvscsi by default (#522) CP-4471 [Backport of CP-4451 to 6.0.7.0] Bump OVA hardware version to HWv11 * Backporting #523, #524, #525, and #526. (#527) * Backporting #523, #524, #525, and #526. (#528) * Fix branch.config * DLPX-71882 [Backport of DLPX-71833 to 6.0/stage] ui-precommit fails to start ChromeHeadless (#530) * Initial changes to support new hotfix workflow(s) (#507) (#531) * Support "hotfix-version" property on build and upgrade * Support hotfix version in upgrade image * Set hotfix version via "execute" script Co-authored-by: Prakash Surya * DLPX-75090 [Backport of DLPX-75089 to 6.0.8.0] Upgrade verification should not enable nfs services (#535) * DLPX-75232 [Backport of DLPX-75229 to 6.0.8.0] finalize() should not rely on properties in upgrade.properties (#539) * DLPX-75258 [Backport of DLPX-73923 to 6.0.9.0] Persist iSCSI initiator files during not-in-place upgrade (#542) * DLPX-75450 [Backport of DLPX-75347 to 6.0.9.0] old rootfs versions are not removed on upgrade in 6.0.8.0 (#545) * DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547) * DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549) * Don't remove "delphix-entire" when generating image diffs (#536) (#537) Currently, if we generate an upgrade image diff from two images that have the same delphix-entire package version, the resultant Aptly repository will not contain any delphix-entire package in it. This is problematic when that Aptly repository is later used with the script used to generate an upgrade image from that repository, as the script requires the repository to contain a "delphix-entire" package in it. Thus, to avoid this issue, this change modifies the image diff logic to ensure we never remove the "delphix-entire" package. * DLPX-75091 DLPX-75122 Make upgrade image prepare script hotfix aware (#543) (#550) DLPX-75091 Make upgrade image prepare script hotfix aware DLPX-75122 Use delphix version and hotfix in version.info in verify-jar script * Revert "DLPX-68959 [Backport of DLPX-64201 to 6.0.9.0] determine how to remove old linux kernels (#549)" (#555) This reverts commit f6421aa1f2112ec17eb53737d079ff87f1e1a533. * Revert "DLPX-75578 [Backport of DLPX-75576 to 6.0.9.0] disable upgrade verification for hotfixes applied via upgrade-scripts (#547)" (#556) This reverts commit 26c960f1697c1cef7d66bae8326670094b9e206d. * DLPX-76243 management stack failed to come back after rollback upgrade (#558) * CP-5356 [Backport of CP-5334 to 6.0.9.0] Add ufw package to appliance-build (#559) * DLPX-76329 [Backport of DLPX-76328 to 6.0.9.0] upgrade script obtains hotfix version incorrectly (#564) In commit e7dd85e2 we introduced the `get_hotfix_version` function for obtaining the hotfix version of the currently running root filesystem. The problem is that in c4357a4e we added logic that intended to use this function, but instead tries to call the `get_current_hotfix` function, which doesn't exist. This leads us to never retreiving the hotfix version correctly via the `rollback` subcommand of the `upgrade` script. * DLPX-74045 adapt upgrade "execute" script to better handle new hotfix workflow (#520) (#568) Currently when applying a hotfix image generated by the new hotfix workflow being developed in CP-4190, nothing will happen when the upgrade is attempted. This is due to the "delphix-entire" package in the hotfix being installed, matching the version of the package that's already installed. Thus, the package manager chooses not to install the new packagecontained in the upgrade/hotfix image, since it thinks the same package is already installed, due to the two having the same version. We need to adapt the "execute" script that's used during upgrade, such that we always install the new "delphix-entire" package contained in the upgrade/hotfix image, even if the package versions are identical. * TOOL-11824 Harden against mistakes when running via Jenkins (#569) (#570) This change adds logic to cause a failure when certain environment variables are missing, and the build is being run via Jenkins. When environment variables are missing, we currently will provide default values, such that the build doesn't fail. These default values generally are not sufficient for anything besides allowing the build to complete (e.g. the resultant upgrade images may fail upgrade). Further, the main motivation for generating these default values, is to make it easier to run the build manually, and aid developer iteration. Thus, when it's clear that the build is being run via Jenkins, where these missing environment variables more likely point to an error or broken automation, than an intentional omission of these variables, it's better to report the error than set and use a default value. This way, we can more quickly detect and address the problem, than silently produce (potentially) incorrect or broken artifacts. * [Backport to 6.0] Remove code for building migration images (#571) Co-authored-by: John Gallagher * DLPX-76433 [Backport of DLPX-76376 to 6.0.10.0] rootfs property "com.delphix:hotfix-version" incorrect after rollback of hotfix (#573) * DLPX-76478 [Backport of DLPX-71683 to 6.0.10.0] Generate new buildinfo package using metadata from combined packages (#493) (#572) * Add rust to dev environment (#544) (#577) Co-authored-by: Paul Dagnelie * Backport a few changes from master to 6.0/stage (#579) * remove crash-python role from appliance-build Now that we can run crash-python outside the git repo from any path and we are halting future development on it, its role on the appliance build is no longer needed for developer VMs. Note that deleting that role uncovered a different bug where the minimal-common role in internal-dev is needed by the ldap role and it would always run before it because it was listed as a dependency under the crash python role. Now that the crash-python role is removed we re-add an old version of that configuration making the minimal-common role run before the ldap role. * Add "shell" subcommand to upgrade-container script (#420) This adds a new "shell" subcommand to the "upgrade-container" script, which is intended to enable users to obtain an interactive shell session within the specified container. Currently, the "run" subcommand can be used, while specifying the shell to run (e.g. /bin/bash), but since that does not specify the "--pty" option with "systemd-run", the terminal may not work correctly (e.g. keyboard input, pager output, etc). Now, when folks want to obtain an interactive shell, they should use this new "shell" subcommand, rather than "run". * Fix "upgrade-container shell" argument handling (#426) Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Prakash Surya * DLPX-76770 [Backport of DLPX-68846] logic for determining device path for grub operations during upgrade is fragile (#581) * DLPX-76769 [Backport of DLPX-76694 to 6.0.10.0] Failure to build DelphixConnector on Delphix Engine, causing gui_sanity testing to fail (#580) * DLPX-76293 Copy Hotfix Metadata Generation to appliance-build (#574) (#590) Co-authored-by: Matt Skinner * install-ansible-lint is failing (#587) (#593) Co-authored-by: Pavel Zakharov * DLPX-76893 [Backport of DLPX-76802 to 6.0.10.0] Starting Verification container enables IP forwarding on host (#592) * Want variant-specific package lists for `live-build(7)` (#596) * DLPX-77250 [Backport of DLPX-74852 to 6.0.11.0] Upgrade subsystem leaves behind stale entry in /etc/apt/sources.list (#602) * TOOL-12103 [Backport of TOOL-12101 to 6.0/stage] ui-precommit fails with missing necessary HEADLESS chrome dependency (#604) * Use "delphix-rust" instead of Ubuntu's Rust packages (#606) * CP-5971 [Backport of CP-5946 to 6.0.11.0] Enable Java debug mode for sso app (#610) * DLPX-77019 [Backport of DLPX-76998 to 6.0.11.0] CLONE - Starting Verification container enables IP forwarding on host (#611) * DLPX-76940 [Backport of DLPX-76907 to 6.0.11.0] obsolete conf file cleanup logic incorrectly removes conf files that were moved to another package (#613) * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12434 Need to use devops-bot token to fetch some repositories during appliance-build * TOOL-12005 Enable ntp on dcenter-internal variant (#594) * Use HTTPS URL for Artifactory (#598) * DLPX-77385 Upgrade: all packages should be upgraded at the same time, remove exception for delphix-platform (#608) * TOOL-12324 appliance-build: add a retry when unmount fails with EBUSY (#617) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab (#619) * TOOL-12062 Appliance-build sometimes fails on focal branch when fetching git repos from gitlab * fixes * DLPX-77878 Fix debootstrap arguments when creating not-in-place upgrade container (#616) * TOOL-12323 Remove libnss3-dbg package as it is missing on Ubuntu 20.04 (#618) * QI-3191 dlpx-qa-gate not installing on DCoLs as Jenkins agents (#623) Co-authored-by: Brandon Lim * TOOL-12368 appliance-build: update some python2 package dependencies to python3 (#621) * DLPX-78255 ui-precommit fails due to missing Chrome dependency libnss3.so (#624) * DLPX-78304 [Backport of DLPX-77901 to 6.0.12.0] Prevent services from being re-enabled on upgrade (#627) * DLPX-78328 [Backport of DLPX-78275 to 6.0.12.0] Upgrade-verification is failing because /sys/module is not available in container (#629) * DLPX-78436 [Backport of DLPX-78371 to 6.0.12.0] Upgrade fails because some packages are unexpectedly autoremoved by apt (#635) Co-authored-by: Pavel Zakharov * DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636) * TOOL-12541 [Backport of TOOL-12471 to 6.0.12.0] Remove Jenkins job references to devops-gate/master in appliance-build (#637) * Don't use "apt-get autoremove" anymore (#642) (#643) We've discovered that "apt-get autoremove" does not work for our needs. Specifically, packages may erroneously remain installed due to suggested and recommended package dependencies, as well as due to virtual package dependencies. While we could alleviate the issue w.r.t. suggested and recommended dependencies via configuration changes, there isn't a clear way to resolve the issue for virtual packages. Thus, the solution adopted in this change is to simply not use "apt-get automoremove" anymore, and instead, explicitly "apt-get purge" all package that we intend to be removed. This addresses the following two bugs: * DLPX-64201 determine how to remove old linux kernels * DLPX-78220 Old packages not removed after deferred upgrade * DLPX-78307 Upgrade tests to trunk failed (#647) (#649) * DLPX-77974 Add minimum version property to root dataset (#620) (#650) * TOOL-12418 [Backport of TOOL-12369 to 6.0.13.0] appliance-build: changes for Ubuntu 20.04 (#653) * CP-7160 [Backport of CP-6676 to 6.0.13.0] appliance-build changes to simplify the fluentd build (#655) * DLPX-79489 [Backport of DLPX-79367] td-agent service is not masked after upgrade (#659) * DLPX-79276 [Backport of DLPX-78736 to 6.0.13.0] Several test_api_and_cli_list_operation tests failed after stage to trunk upgrade (#660) * Rely on pre-installed "awscli" package (#661) (#662) * DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670) * DLPX-79030 [Backport of DLPX-78464 to 6.0.14.0] Regression in appliance-build from DLPX-76293 (#669) * Revert "DLPX-80424 [Backport of DLPX-80287 to 6.0.14.0] remove unnecessary "recommended" packages from appliance (#670)" (#672) This reverts commit 0d0cf2cee47ea192cd6f7f4c2e7d904654724c55. * DLPX-80654 [Backport of DLPX-80114 to 6.0.14.0] Add list of installed linux packages for each platform variant during appliance build (#663) (#674) * TOOL-13514 [Backport of TOOL-13469 to 6.0.14.0] appliance-build changes to add telegraf support (#675) * Revert "DLPX-78582 6.0/stage build needs python3.8 in order to release vSDK 4.0.0 with 6.0.12.0 (appliance-build) (#636)" This reverts commit 74543b0c46d2f9a8f85e1cb33dc199d02d447bb5. * DLPX-80424 [Backport of DLPX-80287 to 6.0.15.0] remove unnecessary "recommended" packages from appliance * DLPX-80917 [Backport of DLPX-80873 to 6.0.15.0] Upgrade verification gets stuck when upgrading from 6.0.13.0 -> 6.0.13.1 (#677) * CP-8412 [Backport of CP-8404 to 6.0.15.0] disable default telegraf service (#684) * QA-35796 [Backport of QA-35550 to 6.0.15.0] Add nmap to qa-internal appliance builds (#679) * DLPX-75209 -- Fix for upgrade scenario * DLPX-75209 rebooting a Delphix Engine in AWS can leave the system with no network configuration * Update execute Update the execute script to follow guidelines of shfmt script. * Update execute Incorporate changes suggested by the shellcheck script * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested by the shfmt script. Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> * Update execute Fix syntax of find command in function generate_interface_to_mac_address_map * Update execute Simplify the logic in generate_interface_to_mac_address_map method through the following changes -- 1. Allow an entry for the loopback interface in the interface <-> mac address map file. This presence of the loopback interface in the map file is harmless and simplifies the logic. 2. Skip the truncation of ":" characters in the mac_address. This truncation will executed in the script /usr/bin/update_netplan_to_mac_address which is managed by the delphix-platform repository and updates the netplan file to mac address based interface names while booting into the upgraded image post engine upgrade. * Update execute Incorporate changes suggested by the shfmt script. * Update execute Incorporate changes suggested in the review. 1. Removed the check for "rollback" type before calling the "generate_interface_to_mac_address_map" function. 2. Changed the target location for the map file from /etc/ to /var/tmp/delphix-upgrade * Update execute Removed the check for "rollback" type before calling the generate_interface_to_mac_address_map function. * Update execute Moved the definition of INTERFACE_TO_MACADRESS_MAP_FILE_PATH variable inside the definition of generate_interface_to_mac_address_map() function. * Update execute Modify the location of the interface to mac address map file from /var/tmp/delphix-upgrade to /etc. This is being done because we found out in testing that "/var/tmp/delphix-mgmt/interface_to_macaddress_map.out" is not available (likely does not exist) when the systemd-networkd daemon is started in the initramfs phase while booting up into the new image post upgrade. The "/etc/interface_to_macaddress_map.out" path is available. It seems this is a result of only specific paths getting built into the initramfs image. Co-authored-by: Delphix User Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> * Fix permissions on the execute script -- Revert back to 0755 from 0644 * Remove the global declaration of INTERFACE_TO_MACADRESS_MAP_FILE_PATH in execute script Co-authored-by: Delphix User Co-authored-by: Garrett Rodriguez Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com> Co-authored-by: John Gallagher Co-authored-by: grodr <39814162+grodr@users.noreply.github.com> Co-authored-by: Prakash Surya Co-authored-by: sara hartse Co-authored-by: Tony Nguyen Co-authored-by: Pavel Zakharov Co-authored-by: Sebastien Roy Co-authored-by: George Wilson Co-authored-by: John Gallagher Co-authored-by: AdityaPrasad-dlpx <54638411+AdityaPrasad-dlpx@users.noreply.github.com> Co-authored-by: Sanjay Bharadwaj Co-authored-by: Don Brady Co-authored-by: Paul Dagnelie Co-authored-by: Raul Santelices Co-authored-by: Basil Crow Co-authored-by: brandon.lim Co-authored-by: mneetha Co-authored-by: Neetha Menon Co-authored-by: Paul Dagnelie Co-authored-by: Kai Chen <50645101+ckdelphix@users.noreply.github.com> Co-authored-by: Matt Skinner Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Co-authored-by: Sravya Meda Co-authored-by: Manoj Joseph Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Raul Santelices Co-authored-by: Brandon T. Lim Co-authored-by: mrburke Co-authored-by: jorge-delphix <82233062+jorge-delphix@users.noreply.github.com> Co-authored-by: Brad Lewis Co-authored-by: Neal Quigley Co-authored-by: Sebastien Roy Co-authored-by: vimalraj-anbarasu <51975401+vimalraj-anbarasu@users.noreply.github.com> Co-authored-by: Akash Ayare <102787258+aayare-dlpx@users.noreply.github.com> From 701ba8d98721357ed67d662ee846d39a938d7408 Mon Sep 17 00:00:00 2001 From: Naveen Singh Date: Mon, 19 Sep 2022 08:30:11 +0530 Subject: [PATCH 154/197] Fix permissions on the execute script -- Revert back to 0755 from 0644 --- upgrade/upgrade-scripts/execute | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 upgrade/upgrade-scripts/execute diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute old mode 100644 new mode 100755 From 1c0e7efb41da1317eff81a5d3b3f58d3f92c14b3 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 26 Sep 2022 11:48:40 -0700 Subject: [PATCH 155/197] Create new variant for linux package mirror (#691) = Context DevOps asked for a new variant which consists of a plain Ubuntu system with zfs installed similar to DCenter but without extraneous stuff (NFS, LDAP, specific DHCP configuration, etc..). The variant will be used to be the point of access of our linux package mirror backed by S3. The variant will be accessing S3 using the zfs-object-agent which will mount the package mirror data stored in S3 as an object-store pool. For more info refer to http://reviews.delphix.com/r/83379/. = This Patch Adds the new variant called `internal-package-mirror` which is an Ubuntu system with the following features: - A `delphix` user with sudo priviledges - Delphix kernel bits including ZFS and its userland components - The devops-gate cloned under `/opt/devops-gate`. = Side-Changes Update README.md --- README.md | 6 ++-- .../tasks/main.yml | 32 +++++++++++++++++++ .../ansible/playbook.yml | 25 +++++++++++++++ .../internal-package-mirror/ansible/roles | 1 + .../package-lists/package-mirror.list.chroot | 22 +++++++++++++ 5 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml create mode 100644 live-build/variants/internal-package-mirror/ansible/playbook.yml create mode 120000 live-build/variants/internal-package-mirror/ansible/roles create mode 100644 live-build/variants/internal-package-mirror/package-lists/package-mirror.list.chroot diff --git a/README.md b/README.md index 6875d81a..1574bf69 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ EC2, Microsoft Azure, VMware, OpenStack). Run this command on "dlpxdc.co" to create the VM used to do the build: - $ dc clone-latest --size COMPUTE_LARGE bootstrap-18-04 $USER-bootstrap + $ dc clone-latest --size COMPUTE_LARGE bootstrap-20-04 $USER-bootstrap Log into that VM using the "ubuntu" user, and run these commands: @@ -52,7 +52,7 @@ images. ### Step 1: Create Docker Host using DCenter on AWS -Delphix maintains the "bootstrap-18-04" group in DCenter on AWS that +Delphix maintains the "bootstrap-20-04" group in DCenter on AWS that fulfills the required build dependencies previously described. Thus, the first step is to use this group to create the host that will be used to execute the build. This can be done as usual, using "dc clone-latest". @@ -60,7 +60,7 @@ execute the build. This can be done as usual, using "dc clone-latest". Example commands running on "dlpxdc.co": $ export DLPX_DC_INSTANCE_PUB_KEY=~/.ssh/id_rsa.pub - $ dc clone-latest --size COMPUTE_LARGE bootstrap-18-04 ps-build + $ dc clone-latest --size COMPUTE_LARGE bootstrap-20-04 ps-build Use the "ubuntu" user to log in to the VM after it's cloned; all of the following steps assume their being run on the cloned VM. diff --git a/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml new file mode 100644 index 00000000..5e0bd447 --- /dev/null +++ b/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml @@ -0,0 +1,32 @@ +# +# Copyright 2022 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +# +# We are wrapping this in a retry block because the Gitlab server sometimes +# hangs up unexpectedly. We haven't seen this with GitHub, so we should +# probably remove the retry once we switch to GitHub. +# +- git: + repo: 'https://gitlab.delphix.com/devops/devops-gate.git' + version: master + dest: /opt/devops-gate + accept_hostkey: yes + update: no + retries: 3 + delay: 30 + register: result + until: result is not failed diff --git a/live-build/variants/internal-package-mirror/ansible/playbook.yml b/live-build/variants/internal-package-mirror/ansible/playbook.yml new file mode 100644 index 00000000..fd02f513 --- /dev/null +++ b/live-build/variants/internal-package-mirror/ansible/playbook.yml @@ -0,0 +1,25 @@ +# +# Copyright 2022 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +- hosts: all + gather_facts: no + vars: + ansible_python_interpreter: /usr/bin/python3 + roles: + - appliance-build.minimal-common + - appliance-build.minimal-internal + - appliance-build.devops-development diff --git a/live-build/variants/internal-package-mirror/ansible/roles b/live-build/variants/internal-package-mirror/ansible/roles new file mode 120000 index 00000000..58b96301 --- /dev/null +++ b/live-build/variants/internal-package-mirror/ansible/roles @@ -0,0 +1 @@ +../../../misc/ansible-roles \ No newline at end of file diff --git a/live-build/variants/internal-package-mirror/package-lists/package-mirror.list.chroot b/live-build/variants/internal-package-mirror/package-lists/package-mirror.list.chroot new file mode 100644 index 00000000..e4906d98 --- /dev/null +++ b/live-build/variants/internal-package-mirror/package-lists/package-mirror.list.chroot @@ -0,0 +1,22 @@ +# +# Copyright 2022 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +git +python3 +python3-dbg +python3-dev +python3-pip +python3-venv From 9d9fe1d7b1b9bfad28b81250a18942488770d42e Mon Sep 17 00:00:00 2001 From: "saurabh.rusia" Date: Tue, 11 Oct 2022 23:53:32 +0530 Subject: [PATCH 156/197] SCAL-738 Setup creation fails trying to install filebeat agent on dlpx-qa images as gnupg installation fails PR URL: https://www.github.com/delphix/appliance-build/pull/695 --- .../ansible-roles/appliance-build.qa-internal/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml index 2a6c88ed..ac82afbc 100644 --- a/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml @@ -24,6 +24,7 @@ - snmptrapd - ufw - nmap + - gnupg state: present register: result until: result is not failed From 60a50c5674dd64b57d713808cd07b6f645e5c8ef Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 11 Oct 2022 11:38:32 -0700 Subject: [PATCH 157/197] TOOL-15951 hotfixes without changes fail to build sparse hotfix image (#693) --- scripts/aptly-repo-from-image-diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aptly-repo-from-image-diff.sh b/scripts/aptly-repo-from-image-diff.sh index 80891dd3..2e1e37b7 100755 --- a/scripts/aptly-repo-from-image-diff.sh +++ b/scripts/aptly-repo-from-image-diff.sh @@ -131,7 +131,7 @@ aptly repo search image-b | xargs aptly repo remove upgrade-repository || die "failed to remove packages from repository: 'upgrade-repository'" -aptly publish repo -skip-contents -skip-signing upgrade-repository || +aptly publish repo -skip-contents -skip-signing -architectures=amd64 upgrade-repository || die "failed to publish repository: 'upgrade-repository'" [[ -d ~/.aptly/public ]] || die "failed to generate aptly repository" From 5ffe55d43729711908e887c8e909745a37a3a6ba Mon Sep 17 00:00:00 2001 From: Sebastien Roy Date: Wed, 2 Nov 2022 20:58:15 -0400 Subject: [PATCH 158/197] TOOL-11972 appliance-build on focal: cannot export 'rpool': pool is busy (#698) TOOL-11972 appliance-build on focal: cannot export 'rpool': pool is busy PR URL: https://www.github.com/delphix/appliance-build/pull/698 --- .../vm-artifacts/90-raw-disk-image.binary | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index feb5706f..3b1b9e09 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -27,6 +27,23 @@ die() { exit 1 } +# +# usage: retry command arg1 arg2 ... +# +retry() { + attempts=$1 + shift + delay=$1 + shift + + for attempt in $(seq "${attempts}"); do + "$@" && break + [[ $attempt == "$attempts" ]] && die "Too many failed attempts, aborting." + echo "Attempt $attempt failed, trying again after a small nap." + sleep "$delay" + done +} + # # The root filesystem container needs to have the appliance version # embedded in it as a dataset property, thus if this value was not @@ -305,18 +322,13 @@ chroot "$DIRECTORY" grub-mkconfig -o /mnt/boot/grub/grub.cfg chroot "$DIRECTORY" umount /mnt for dir in /dev /proc /sys; do - for attempt in {1..5}; do - umount -R "${DIRECTORY}${dir}" && break - [[ "$attempt" == 5 ]] && die "Too many failed attempts, aborting." - echo "Attempt $attempt failed, trying again after a small nap." - sleep 10 - done + retry 5 10 umount -R "${DIRECTORY}${dir}" done umount "$DIRECTORY/var/log" umount "$DIRECTORY/var/delphix" umount "$DIRECTORY/export/home" umount "/var/crash" -zfs umount "rpool/ROOT/$FSNAME/root" -zpool export rpool +retry 5 10 zfs umount "rpool/ROOT/$FSNAME/root" +retry 5 10 zpool export rpool kpartx -d "$ARTIFACT_NAME.img" From 7d510dc56ffc4427a9aec614b1e36183e85cc8c5 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 4 Nov 2022 08:17:19 -0700 Subject: [PATCH 159/197] TOOL-16450 Support running on a ZFS-root based buildserver (#700) --- .../vm-artifacts/90-raw-disk-image.binary | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index 3b1b9e09..d61b1d25 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -139,38 +139,39 @@ zpool create -d \ -O mountpoint=none \ -O compression=on \ -R "$DIRECTORY" \ + -t "$FSNAME" \ rpool "/dev/mapper/${LOOPNAME}p1" zfs create \ -o canmount=off \ -o mountpoint=none \ - "rpool/ROOT" + "$FSNAME/ROOT" zfs create \ -o canmount=off \ -o mountpoint=none \ -o "com.delphix:initial-version=$DELPHIX_APPLIANCE_VERSION" \ -o "com.delphix:current-version=$DELPHIX_APPLIANCE_VERSION" \ - "rpool/ROOT/$FSNAME" + "$FSNAME/ROOT/$FSNAME" if [[ -n "$DELPHIX_HOTFIX_VERSION" ]]; then zfs set \ "com.delphix:hotfix-version=$DELPHIX_HOTFIX_VERSION" \ - "rpool/ROOT/$FSNAME" + "$FSNAME/ROOT/$FSNAME" fi if [[ -n "$DELPHIX_MINIMUM_VERSION" ]]; then zfs set \ "com.delphix:minimum-version=$DELPHIX_MINIMUM_VERSION" \ - "rpool/ROOT/$FSNAME" + "$FSNAME/ROOT/$FSNAME" fi zfs create \ -o canmount=noauto \ -o mountpoint=/ \ - "rpool/ROOT/$FSNAME/root" + "$FSNAME/ROOT/$FSNAME/root" -zfs mount "rpool/ROOT/$FSNAME/root" +zfs mount "$FSNAME/ROOT/$FSNAME/root" # # We are later going to recursively bind mount /proc/, /sys/, and /dev/ @@ -205,15 +206,15 @@ mount --make-slave "$DIRECTORY" zfs create \ -o mountpoint=legacy \ - "rpool/ROOT/$FSNAME/home" + "$FSNAME/ROOT/$FSNAME/home" zfs create \ -o mountpoint=legacy \ - "rpool/ROOT/$FSNAME/data" + "$FSNAME/ROOT/$FSNAME/data" zfs create \ -o mountpoint=legacy \ - "rpool/ROOT/$FSNAME/log" + "$FSNAME/ROOT/$FSNAME/log" # # Initialize the grub dataset. This dataset will be used to contain all @@ -234,7 +235,7 @@ zfs create \ # zfs create \ -o mountpoint=legacy \ - rpool/grub + "$FSNAME/grub" # # Initialize the crashdump dataset. This is used to store core files @@ -244,8 +245,8 @@ zfs create \ # zfs create \ -o mountpoint=legacy \ - -o quota="$(echo "$(zpool list -Hpo size rpool) / 2" | bc)b" \ - rpool/crashdump + -o quota="$(echo "$(zpool list -Hpo size "$FSNAME") / 2" | bc)b" \ + "$FSNAME/crashdump" # # Since these datasets use "legacy" for their mountpoints, we need to @@ -254,16 +255,16 @@ zfs create \ # these mounts. # mkdir -p "$DIRECTORY/export/home" -mount -t zfs "rpool/ROOT/$FSNAME/home" "$DIRECTORY/export/home" +mount -t zfs "$FSNAME/ROOT/$FSNAME/home" "$DIRECTORY/export/home" mkdir -p "$DIRECTORY/var/delphix" -mount -t zfs "rpool/ROOT/$FSNAME/data" "$DIRECTORY/var/delphix" +mount -t zfs "$FSNAME/ROOT/$FSNAME/data" "$DIRECTORY/var/delphix" mkdir -p "$DIRECTORY/var/log" -mount -t zfs "rpool/ROOT/$FSNAME/log" "$DIRECTORY/var/log" +mount -t zfs "$FSNAME/ROOT/$FSNAME/log" "$DIRECTORY/var/log" mkdir -p "/var/crash" -mount -t zfs "rpool/crashdump" "/var/crash" +mount -t zfs "$FSNAME/crashdump" "/var/crash" # # Populate the root filesystem with the contents of the "binary" directory @@ -316,7 +317,7 @@ done # We need to use the dedicated grub dataset when running "grub-install" # and "grub-mkconfig", so we need to mount this dataset first. # -chroot "$DIRECTORY" mount -t zfs rpool/grub /mnt +chroot "$DIRECTORY" mount -t zfs "$FSNAME/grub" /mnt chroot "$DIRECTORY" grub-install --root-directory=/mnt "/dev/$LOOPNAME" chroot "$DIRECTORY" grub-mkconfig -o /mnt/boot/grub/grub.cfg chroot "$DIRECTORY" umount /mnt @@ -329,6 +330,6 @@ umount "$DIRECTORY/var/log" umount "$DIRECTORY/var/delphix" umount "$DIRECTORY/export/home" umount "/var/crash" -retry 5 10 zfs umount "rpool/ROOT/$FSNAME/root" -retry 5 10 zpool export rpool +retry 5 10 zfs umount "$FSNAME/ROOT/$FSNAME/root" +retry 5 10 zpool export "$FSNAME" kpartx -d "$ARTIFACT_NAME.img" From 6f36bf91ba6e44c1330eb5907b6c175b49471b8f Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 4 Nov 2022 11:55:53 -0700 Subject: [PATCH 160/197] TOOL-16451 Add new "buildserver" variant (#699) This change adds a new "buildserver" variant, where the intention is to use this variant for building our software (packages, appliance, etc), rather than our current buildserver image which is based on Ubuntu. --- .../tasks/main.yml | 24 +++++++++++++++++ .../internal-buildserver/ansible/playbook.yml | 26 +++++++++++++++++++ .../internal-buildserver/ansible/roles | 1 + 3 files changed, 51 insertions(+) create mode 100644 live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml create mode 100644 live-build/variants/internal-buildserver/ansible/playbook.yml create mode 120000 live-build/variants/internal-buildserver/ansible/roles diff --git a/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml new file mode 100644 index 00000000..8d2ddf07 --- /dev/null +++ b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml @@ -0,0 +1,24 @@ +# +# Copyright 2022 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +- apt: + name: + - adoptopenjdk-java8-jdk + - curl + - git + - gnupg + state: present diff --git a/live-build/variants/internal-buildserver/ansible/playbook.yml b/live-build/variants/internal-buildserver/ansible/playbook.yml new file mode 100644 index 00000000..49332873 --- /dev/null +++ b/live-build/variants/internal-buildserver/ansible/playbook.yml @@ -0,0 +1,26 @@ +# +# Copyright 2022 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +- hosts: all + gather_facts: no + vars: + ansible_python_interpreter: /usr/bin/python3 + roles: + - appliance-build.minimal-common + - appliance-build.minimal-internal + - appliance-build.minimal-development + - appliance-build.buildserver-internal diff --git a/live-build/variants/internal-buildserver/ansible/roles b/live-build/variants/internal-buildserver/ansible/roles new file mode 120000 index 00000000..58b96301 --- /dev/null +++ b/live-build/variants/internal-buildserver/ansible/roles @@ -0,0 +1 @@ +../../../misc/ansible-roles \ No newline at end of file From a283e659f341530a49c8fb1bdadc0031bf88bfac Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 7 Nov 2022 12:16:08 -0800 Subject: [PATCH 161/197] TOOL-16514 Add docker.io package to buildserver variant (#705) --- .../appliance-build.buildserver-internal/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml index 8d2ddf07..cf7ec527 100644 --- a/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml @@ -19,6 +19,11 @@ name: - adoptopenjdk-java8-jdk - curl + - docker.io - git - gnupg state: present + +- user: + name: delphix + groups: docker From 340ba84685c70b593e337e82456ee7a5abc4dcba Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 9 Nov 2022 11:10:25 -0800 Subject: [PATCH 162/197] TOOL-16594 Don't remove delphix user from existing groups (#706) --- .../appliance-build.buildserver-internal/tasks/main.yml | 1 + .../ansible-roles/appliance-build.minimal-common/tasks/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml index cf7ec527..b6f81ec3 100644 --- a/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml @@ -27,3 +27,4 @@ - user: name: delphix groups: docker + append: true diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index fcb8a962..9bb70ec9 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -35,6 +35,7 @@ uid: 65433 group: staff groups: root + append: true shell: /bin/bash create_home: yes comment: Delphix User From fcb0f101dc6206d002dc6cef206c9b0c1e84d5cc Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 9 Nov 2022 11:24:15 -0800 Subject: [PATCH 163/197] TOOL-16510 Add more rpool space for buildserver variant (#704) --- .../hooks/vm-artifacts/90-raw-disk-image.binary | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary index d61b1d25..fd867edd 100755 --- a/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary +++ b/live-build/config/hooks/vm-artifacts/90-raw-disk-image.binary @@ -60,15 +60,22 @@ retry() { # We want to use different sized rpool depending on if we're building a # disk image meant for internal use, or external (i.e. customer) use. # -# The only exception to this is our "dcenter" variant. While that -# variant is only used internally, we use it in a way that more -# resembles our external variants, so we want the rpool size for -# the dcenter images to match our external images. +# One exception to this is our "dcenter" variant. While that variant is +# only used internally, we use it in a way that more resembles our +# external variants, so we want the rpool size for the dcenter images to +# match our external images. +# +# Another exception is our "buildserver" variant. We use those images to +# generate our appliance images, and upgrade images, which consume a lot +# of space during the build process; thus, we need a larger rpool. # case "$APPLIANCE_VARIANT" in external-* | internal-dcenter) RAW_DISK_SIZE_GB=127 ;; +internal-buildserver) + RAW_DISK_SIZE_GB=256 + ;; internal-*) RAW_DISK_SIZE_GB=70 ;; From 87f201cf015b341bffd1af34392b9d7afe0a7db4 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Sat, 3 Dec 2022 18:14:50 -0800 Subject: [PATCH 164/197] DLPX-83408 Post-upgrade cleanup fails with internal error due to already deleted dataset (#696) --- upgrade/upgrade-scripts/common.sh | 8 +++++++- upgrade/upgrade-scripts/rootfs-container | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index ba24ea65..c52058eb 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -169,7 +169,13 @@ function get_dataset_rollback_snapshot_name() { } function get_snapshot_clones() { - zfs get clones -Hpo value "$1" + local CLONES + + CLONES="$(zfs get clones -Hpo value "$1")" + + if [[ "$CLONES" != "-" ]]; then + echo "$CLONES" + fi } function get_version_property() { diff --git a/upgrade/upgrade-scripts/rootfs-container b/upgrade/upgrade-scripts/rootfs-container index 9789531a..bd384010 100755 --- a/upgrade/upgrade-scripts/rootfs-container +++ b/upgrade/upgrade-scripts/rootfs-container @@ -23,8 +23,10 @@ function delete() { zfs list "rpool/ROOT/$CONTAINER/root" &>/dev/null || die "rootfs container '$CONTAINER' does not exist" + MOUNTPOINT=$(zfs get mountpoint -Hpo value "rpool/ROOT/$CONTAINER/root") MOUNTED=$(zfs get mounted -Hpo value "rpool/ROOT/$CONTAINER/root") - [[ "$MOUNTED" == "no" ]] || + + [[ "$MOUNTPOINT" == "/" ]] && [[ "$MOUNTED" == "yes" ]] && die "cannot delete mounted rootfs container: '$CONTAINER'" local snapname From d7a932e60e6b5cb6e9b70dcaedeafc040efdd934 Mon Sep 17 00:00:00 2001 From: Jules Tamagnan <77420472+jtamagnan-delphix@users.noreply.github.com> Date: Thu, 12 Jan 2023 11:43:40 -0800 Subject: [PATCH 165/197] TOOL-17749 Point devops-gate to GitHub repository (#711) PR URL: https://www.github.com/delphix/appliance-build/pull/711 --- .../tasks/main.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml index 5e0bd447..aafdb9bc 100644 --- a/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml @@ -15,18 +15,10 @@ # --- -# -# We are wrapping this in a retry block because the Gitlab server sometimes -# hangs up unexpectedly. We haven't seen this with GitHub, so we should -# probably remove the retry once we switch to GitHub. -# - git: - repo: 'https://gitlab.delphix.com/devops/devops-gate.git' - version: master + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/devops-gate.git" + version: main dest: /opt/devops-gate accept_hostkey: yes update: no - retries: 3 - delay: 30 - register: result - until: result is not failed + when: lookup('env', 'GITHUB_TOKEN') != '' From 3eae77ddffbca30d5eb4c3e4660af1f8144dfee7 Mon Sep 17 00:00:00 2001 From: Jules Tamagnan <77420472+jtamagnan-delphix@users.noreply.github.com> Date: Thu, 12 Jan 2023 11:47:42 -0800 Subject: [PATCH 166/197] TOOL-17748 Point dms-core-gate to GitHub repository (#710) PR URL: https://www.github.com/delphix/appliance-build/pull/710 --- .../tasks/main.yml | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml index b0711131..065e3d85 100644 --- a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml @@ -23,26 +23,14 @@ retries: 3 delay: 60 -# -# We are wrapping this in a retry block because the Gitlab server sometimes -# hangs up unexpectedly. We haven't seen this with GitHub, so we should -# probably remove the retry once we switch to GitHub. -# - git: - repo: "{{ item.repo }}" + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dms-core-gate.git" dest: - "/export/home/delphix/{{ item.dest }}" - version: "{{ item.version }}" + "/export/home/delphix/dms-core-gate" + version: "6.0/stage" accept_hostkey: yes update: no - with_items: - - { repo: 'https://gitlab.delphix.com/masking/dms-core-gate.git', - version: master, - dest: dms-core-gate } - retries: 3 - delay: 30 - register: result - until: result is not failed + when: lookup('env', 'GITHUB_TOKEN') != '' - file: path: "/export/home/delphix/{{ item }}" From 53be0e193704686fa0c022ba364705750f7f9628 Mon Sep 17 00:00:00 2001 From: Jules Tamagnan <77420472+jtamagnan-delphix@users.noreply.github.com> Date: Thu, 12 Jan 2023 11:51:39 -0800 Subject: [PATCH 167/197] TOOL-17747 Point dcenter-gate to GitHub repository (#709) PR URL: https://www.github.com/delphix/appliance-build/pull/709 --- .../appliance-build.dcenter/tasks/main.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml index edf426a3..7f47e2dc 100644 --- a/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml @@ -15,21 +15,13 @@ # --- -# -# We are wrapping this in a retry block because the Gitlab server sometimes -# hangs up unexpectedly. We haven't seen this with GitHub, so we should -# probably remove the retry once we switch to GitHub. -# - git: - repo: 'https://gitlab.delphix.com/devops/dcenter-gate.git' - version: master + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dcenter-gate.git" + version: main dest: /opt/dcenter/lib/dcenter-gate accept_hostkey: yes update: no - retries: 3 - delay: 30 - register: result - until: result is not failed + when: lookup('env', 'GITHUB_TOKEN') != '' - alternatives: name: java From b7fd7c99909163817226eabeed6b13dd0a345126 Mon Sep 17 00:00:00 2001 From: Jules Tamagnan <77420472+jtamagnan-delphix@users.noreply.github.com> Date: Wed, 18 Jan 2023 09:00:15 -0800 Subject: [PATCH 168/197] TOOL-17750 Point dlpx-app-gate to GitHub repository (#712) PR URL: https://www.github.com/delphix/appliance-build/pull/712 --- .../tasks/main.yml | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index 2f93749e..60eeda9e 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -73,26 +73,13 @@ [Service] Environment=DLPX_PG_DEBUG=true -# -# We are wrapping this in a retry block because the Gitlab server sometimes -# hangs up unexpectedly. We haven't seen this with GitHub, so we should -# probably remove the retry once we switch to GitHub. -# - git: - repo: "{{ item.repo }}" - dest: - "/export/home/delphix/{{ item.dest }}" - version: "{{ item.version }}" + repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dlpx-app-gate.git" + dest: "/export/home/delphix/dlpx-app-gate" + version: "6.0/stage" accept_hostkey: yes update: no - with_items: - - { repo: 'https://gitlab.delphix.com/app/dlpx-app-gate.git', - version: master, - dest: dlpx-app-gate } - retries: 3 - delay: 30 - register: result - until: result is not failed + when: lookup('env', 'GITHUB_TOKEN') != '' - file: path: "/export/home/delphix/{{ item }}" From 53f28e2ed8d8a7f89ee825502097c5f54fefe9c1 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 7 Feb 2023 16:08:25 -0800 Subject: [PATCH 169/197] Bump image for checks; 18.04 image is deprecated (#714) --- .github/workflows/main.yml | 6 +++--- scripts/run-live-build.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 640b0f11..66932d81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,21 +2,21 @@ on: [push, pull_request] jobs: check-ansible: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v1 - run: sudo ./.github/scripts/install-gradle.sh - run: sudo -E ./.github/scripts/install-ansible-lint.sh - run: /opt/gradle-5.1/bin/gradle ansibleCheck check-shellcheck: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v1 - run: sudo ./.github/scripts/install-gradle.sh - run: sudo ./.github/scripts/install-shellcheck.sh - run: /opt/gradle-5.1/bin/gradle shellCheck check-shfmt: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v1 - run: sudo ./.github/scripts/install-gradle.sh diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index cbc2bc71..dd7a9bf3 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -40,9 +40,9 @@ ALL_RUN_TYPE="all" RUN_TYPES="$UPGRADE_RUN_TYPE|$VM_RUN_TYPE|$ALL_RUN_TYPE" case "$3" in -$UPGRADE_RUN_TYPE) ;; -$VM_RUN_TYPE) ;; -$ALL_RUN_TYPE) ;; +"$UPGRADE_RUN_TYPE") ;; +"$VM_RUN_TYPE") ;; +"$ALL_RUN_TYPE") ;; *) echo "Unknown run type '$3'. Must be one of <$RUN_TYPES>" exit 1 From 5ffe97cdbd3a9207bbc794e3f046762be6be8c80 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 7 Feb 2023 16:23:53 -0800 Subject: [PATCH 170/197] DLPX-68852 systemd-run hung after running "execute" script during upgrade (#713) --- upgrade/upgrade-scripts/upgrade-container | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 2924cc54..d2e736a5 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -586,6 +586,15 @@ function destroy() { } function run() { + # + # We've seen cases where systemd-run fails to exit, even after + # it's run the comamnds/script specified. We've "fixed" this in + # the past by executing "systemd daemon-reexec", and retrying + # the upgrade. Thus, to try and avoid the failure to begin with, + # we restart this service, which serves to run "daemon-reexec". + # + systemctl restart systemd-reexec.service + systemd-run \ --machine="$CONTAINER" \ --setenv=CONTAINER="$CONTAINER" \ From 6d8e8972d0a456148fddab6a5be45e38c5fcf3ed Mon Sep 17 00:00:00 2001 From: Don Brady Date: Fri, 10 Mar 2023 08:05:20 -0700 Subject: [PATCH 171/197] DLPX-85081 Avoid removing "rmtab" file during nfs-utils upgrade (#716) --- upgrade/upgrade-scripts/execute | 38 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 0f8f683d..afc38481 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2018 Delphix +# Copyright 2018, 2023 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -160,40 +160,32 @@ fix_and_migrate_services systemctl mask docker.service # -# Older versions (i.e. the 6.0.0.0 release) of the "nfs-kernel-server" -# package had "etab" file delivered as part of the package. Thus, when -# upgrading the package, the existing "etab" file would get replaced -# with a new/empty "etab" file from the new package. +# Older versions (i.e. before 9.0.0.0 release) of the "nfs-kernel-server" +# package had "rmtab" file delivered as part of the package. Thus, when +# upgrading the package, the existing "rmtab" file would get replaced +# with a new/empty "rmtab" file from the new package. # # This file is dynamically modified during runtime, to contain the list -# of currently exported filesystems. Thus, when it's replaced on upgrade -# with an empty file, this results in all exports being unexported; i.e. -# mountd notices that the file is empty, interprets that to mean no -# filesystems should be currently exported, and then unexports all -# previously exported filesystems. +# of active NFSv3 client mounts. Thus, when it's replaced on upgrade +# with an empty file, this can lead to the NFSv3 services being disabled +# while there are still active v3 mounts. # -# This is problematic, since the act of unexporting the filesystems can -# result in client errors, if those exports are being actively used at -# the time of the upgrade/unexport. +# This is problematic, since disabling NFSv3 services can result in +# client errors, if NFSv3 mounts are being actively used at the time of +# the upgrade. # # To avoid this issue, we're modifying the dpkg state, such that dpkg -# will no longer associate the "etab" file with the "nfs-kernel-server" -# package. This way, when the old package is removed, the "etab" file +# will no longer associate the "rmtab" file with the "nfs-kernel-server" +# package. This way, when the old package is removed, the "rmtab" file # will not be automatically manipulated by dpkg. We *must* do this # prior to invoking "apt-get" below, to ensure we remove the association # before upgrading the package. # -# FWIW, we tried to do this via the package's "preinst" package hook, -# but that did not help; the "etab" file was still removed when the old -# package was replaced/upgraded. -# # We can safely remove this once running a version that no longer -# supports upgrading from the 6.0.0.0 release, since that's the only -# release that suffers from this defect; we've dis-associated the "etab" -# file from the "nfs-kernel-server" package in all later releases. +# supports upgrading from releases earlier than 9.0.0.0. # [[ -e /var/lib/dpkg/info/nfs-kernel-server.list ]] && - sed -i '/\/var\/lib\/nfs\/etab/d' /var/lib/dpkg/info/nfs-kernel-server.list + sed -i '/\/var\/lib\/nfs\/rmtab/d' /var/lib/dpkg/info/nfs-kernel-server.list apt_get update || die "failed to update apt sources" From 14910adf1768ad1a361c5b2b5183e5c9bdbc6188 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 20 Mar 2023 13:02:29 -0700 Subject: [PATCH 172/197] DLPX-68852 systemd-run hung after running "execute" script during upgrade (#717) --- upgrade/upgrade-scripts/upgrade-container | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index d2e736a5..e130a0d8 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -593,7 +593,10 @@ function run() { # the upgrade. Thus, to try and avoid the failure to begin with, # we restart this service, which serves to run "daemon-reexec". # - systemctl restart systemd-reexec.service + # This will fail on versions of Delphix that don't yet have + # this service, so we hide any failure to account for that. + # + systemctl restart systemd-reexec.service 2>/dev/null systemd-run \ --machine="$CONTAINER" \ From cca47a3ba58a4bdc6953ebc5036724b8d34238fa Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 6 Apr 2023 10:03:27 -0700 Subject: [PATCH 173/197] TOOL-20585 Add "dwarves" to ZFS build dependency list (#718) --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index 7e600184..8a158400 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -26,6 +26,7 @@ - cppcheck - curl - delphix-rust + - dwarves - emacs-nox - fakeroot - flake8 @@ -40,6 +41,7 @@ - libssl-dev - libtool - libudev-dev + - llvm-12 - lsscsi - mandoc - nfs-kernel-server From c786926fb74a0231554926cdd56208f832253529 Mon Sep 17 00:00:00 2001 From: David Mendez Date: Wed, 12 Apr 2023 14:43:49 -0700 Subject: [PATCH 174/197] DLPX-84191 Bump up the ESX engine's ova to support HW v13 (ESX 6.5) PR URL: https://www.github.com/delphix/appliance-build/pull/719 --- live-build/config/hooks/vm-artifacts/template.ovf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/template.ovf b/live-build/config/hooks/vm-artifacts/template.ovf index 68ff5ddc..54ae0ca2 100644 --- a/live-build/config/hooks/vm-artifacts/template.ovf +++ b/live-build/config/hooks/vm-artifacts/template.ovf @@ -42,7 +42,7 @@ Virtual Hardware Family 0 @@VM_NAME@@ - vmx-11 + vmx-13 hertz * 10^6 @@ -151,7 +151,7 @@ A human-readable annotation - Delphix Appliance, VM Hardware Version 11 + Delphix Appliance, VM Hardware Version 13 From 51d46b090b78b9662a1b63d9e8467d3b3b031e82 Mon Sep 17 00:00:00 2001 From: Palash Gandhi Date: Thu, 13 Apr 2023 12:35:24 -0700 Subject: [PATCH 175/197] DLPX-85650 cutover: Update UPSTREAM_BRANCH for pregitflow work in appliance-build PR URL: https://www.github.com/delphix/appliance-build/pull/720 --- branch.config | 2 +- .../appliance-build.masking-development/tasks/main.yml | 2 +- .../appliance-build.virtualization-development/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/branch.config b/branch.config index 9e572e67..714ae614 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/stage" +UPSTREAM_BRANCH="develop" diff --git a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml index 065e3d85..9fde6f6d 100644 --- a/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml @@ -27,7 +27,7 @@ repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dms-core-gate.git" dest: "/export/home/delphix/dms-core-gate" - version: "6.0/stage" + version: "develop" accept_hostkey: yes update: no when: lookup('env', 'GITHUB_TOKEN') != '' diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index 60eeda9e..f2458abe 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -76,7 +76,7 @@ - git: repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/dlpx-app-gate.git" dest: "/export/home/delphix/dlpx-app-gate" - version: "6.0/stage" + version: "develop" accept_hostkey: yes update: no when: lookup('env', 'GITHUB_TOKEN') != '' From 78411045850b9e8970a2feac6b3082edf75a846d Mon Sep 17 00:00:00 2001 From: Palash Gandhi Date: Mon, 17 Apr 2023 15:29:51 -0700 Subject: [PATCH 176/197] pre-gitflow cutover requires hardcoded branch to be updated --- branch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/branch.config b/branch.config index 8b13b1e8..39e89b39 100644 --- a/branch.config +++ b/branch.config @@ -10,4 +10,4 @@ # UPSTREAM_BRANCH parameter should be updated by the release lead on branching # -UPSTREAM_BRANCH="6.0/release" +UPSTREAM_BRANCH="release" From 9020f935874fc9a24dcf85749e9d5e6d3fcd9aa8 Mon Sep 17 00:00:00 2001 From: Palash Gandhi Date: Wed, 19 Apr 2023 12:54:01 -0700 Subject: [PATCH 177/197] DLPX-85710 appliance-build: Ansible task checks out outdated branch of ZFS repo on dev variants PR URL: https://www.github.com/delphix/appliance-build/pull/722 --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index 8a158400..40d00578 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -64,7 +64,7 @@ repo: "https://{{ lookup('env', 'GITHUB_TOKEN') }}@github.com/delphix/zfs.git" dest: "/export/home/delphix/zfs" - version: master + version: develop accept_hostkey: yes update: no when: lookup('env', 'GITHUB_TOKEN') != '' From 3b265d12ab0635d12410f5d947614673033103e6 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Wed, 17 May 2023 22:35:31 -0700 Subject: [PATCH 178/197] Add delphix-go to appliance-build (#723) --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index 40d00578..83f22199 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -25,6 +25,7 @@ - build-essential - cppcheck - curl + - delphix-go - delphix-rust - dwarves - emacs-nox From fcbb3c21717e811d03246720a8fad6900cf25c77 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 26 May 2023 14:07:14 -0700 Subject: [PATCH 179/197] DLPX-86152 eliminate debug symbols from the build (#724) --- .../internal-dcenter/package-lists/dcenter.list.chroot | 3 --- 1 file changed, 3 deletions(-) diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index f0f33b9a..9c074cb6 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -22,12 +22,9 @@ gnupg isc-dhcp-server libldap2-dev libsasl2-dev -nfs-common-dbgsym nfs-kernel-server -nfs-kernel-server-dbgsym openjdk-11-jdk-headless python3 -python3-dbg python3-dev python3-ldap python3-marshmallow From e874e53f5664616d3a95ca7d651e06b665af4da0 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Fri, 2 Jun 2023 17:46:16 -0700 Subject: [PATCH 180/197] DLPX-85893 run upgrade "execute" script from separate service (#725) --- upgrade/upgrade-scripts/execute | 130 +++++++++++++++++++--- upgrade/upgrade-scripts/upgrade | 67 +++++------ upgrade/upgrade-scripts/upgrade-container | 15 +-- 3 files changed, 144 insertions(+), 68 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index afc38481..06eb078c 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -24,10 +24,41 @@ set -o pipefail function usage() { echo "$(basename "$0"): $*" >&2 - echo "Usage: $(basename "$0") -p " + echo "Usage: $(basename "$0") [-f] [-p ]" exit 2 } +function post_alert() { + local jmxtool="/opt/delphix/server/bin/jmxtool" + + # + # Alerts are a virtualization service concept. Thus, if we're + # running on a variant that doesn't have the virtualization + # package installed, skip the alert. + # + [[ ! -x "$jmxtool" ]] && return + + # + # Skip the alert when running in an upgrade container, as the + # alert is only meant to notify the user about the host. + # + systemd-detect-virt -qc && return + + # + # The alert isn't critical, so if it fails to post, that's + # acceptiable. Further, it's possible to execute this script + # without the virtualization service running. As a result, we + # use "-w" and don't worry if it returns an error code. + # + if [[ "$1" == "reboot" ]]; then + $jmxtool -w boot upgrade server &>/dev/null + elif [[ "$1" == "restart" ]]; then + $jmxtool -w boot upgrade management &>/dev/null + else + die "invalid alert specified: '$1'" + fi +} + function generate_interface_to_mac_address_map() { INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" @@ -47,17 +78,40 @@ function generate_interface_to_mac_address_map() { die "Failed to generate map from interface names to mac addresses" } -while getopts ':rlBfsp:' c; do +# +# Specifies the platform to upgrade to; by default choose the same +# platform the script is running on. +# +# For not-in-place upgrades, we cannot use the get-appliance-platform +# script to determine the platform, hence why this option exists. +# +# This option should not be used to change platforms via an upgrade. +# +opt_p="" + +# +# Perform a "full" upgrade, which does a system reboot. By default, we +# perform a "deferred" upgrade, which resetarts services, but does not +# reboot the system. +# +opt_f="" + +while getopts ':fp:' c; do case $c in - r | l | B | f | s) ;; # LX-72: For now, silently ignore these. - p) - platform=$OPTARG - ;; + f) eval "opt_$c=true" ;; + p) eval "opt_$c='$OPTARG'" ;; *) usage "illegal options -- $OPTARG" ;; esac done +shift $((OPTIND - 1)) + +[[ $# -ne 0 ]] && usage "too many arguments specified" +[[ "$EUID" -ne 0 ]] && die "must be run as root" -[[ -z "$platform" ]] && usage "platform must be specified" +if [[ -z "$opt_p" ]]; then + opt_p="$(get-appliance-platform)" || + usage "platform must be specified" +fi # # When upgrading the packages on with this script, we want to ensure @@ -205,8 +259,8 @@ apt_get update || die "failed to update apt sources" # by installing delphix-virtualization at a later stage of the build via # ansible hooks, when the delphix-platform package has already been installed. # -if ! dpkg-query -l "delphix-platform-$platform" &>/dev/null; then - apt_get install -y "delphix-platform-$platform" || +if ! dpkg-query -l "delphix-platform-$opt_p" &>/dev/null; then + apt_get install -y "delphix-platform-$opt_p" || die "failed to install delphix-platform" fi @@ -270,7 +324,7 @@ dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || # shellcheck disable=SC2153 apt_get install \ -y --allow-downgrades --reinstall \ - "delphix-entire-$platform=$VERSION" || + "delphix-entire-$opt_p=$VERSION" || die "upgrade failed; from '$CURRENT_VERSION' to '$VERSION'" # @@ -282,17 +336,17 @@ apt_get install \ # particularly in the case of "--reinstall", which replaces the current # package with a new package of the same version (i.e. for hotfixes). # -apt-mark manual "delphix-entire-$platform" || +apt-mark manual "delphix-entire-$opt_p" || die "failed to mark 'delphix-entire' package as 'manual' installed" -[[ -f "/usr/share/doc/delphix-entire-$platform/packages.list.gz" ]] || +[[ -f "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" ]] || die "delphix-entire's packages.list.gz file is missing" -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | xargs_apt_get install -y --allow-downgrades || die "failed to install packages listed in packages.list.gz file" -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | cut -d= -f1 | xargs apt-mark manual || die "failed to mark as manual packages listed in packages.list.gz file" @@ -394,7 +448,7 @@ stop_stderr_redirect_to_system_log # we verify the package is installed and its version is correct; this # is simply to help us be confident that upgrade behaves as we expect. # -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | sed 's/=/ /' | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | sed 's/=/ /' | while read -r name version; do installed=$(dpkg-query -Wf '${Version}' "$name") compare_versions "$installed" "=" "$version" || @@ -497,14 +551,56 @@ fi # The container does not have visibility into the complete network namespace of the # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map -if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then +if ! systemd-detect-virt -qc && [[ "$opt_p" == "aws" ]]; then generate_interface_to_mac_address_map fi +# +# We use a seperate ZFS dataset for GRUB, and this dataset is generally +# not mounted when we update pacakges on the system. Thus, when a new +# kernel package is installed, via the call to "execute" above, the GRUB +# configuration will not be modified to use that new kernel. +# +# In order for the system to use the new kernel after a reboot, we must +# regenerate the GRUB configuration after the new kernel has been +# installed. The "rootfs-container set-bootfs" command will do just +# that; it knows how to mount our GRUB specific dataset, and how +# properly update the GRUB configuration. +# +# Note, we only want to update GRUB when running outside of an upgrade +# container; since executing an upgrade within an upgrade container +# should not affect the host system. +# +if ! systemd-detect-virt -qc; then + ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" + [[ -n "$ROOTFS_CONTAINER" ]] || + die "unable to determine currently mounted rootfs container" + + "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || + die "failed to set-bootfs '$ROOTFS_CONTAINER'" +fi + +systemctl reload delphix-platform.service || + die "failed to reload delphix-platform.service" + # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. # zpool sync rpool || die "'zpool sync rpool' failed" -exit 0 +if [[ -f "$UPDATE_DIR/upgrade.properties" ]]; then + source_upgrade_properties +fi + +if [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; then + post_alert "reboot" + exec systemctl reboot || die "failed to reboot" +else + post_alert "restart" + exec systemctl restart delphix.target || + die "failed to restart delphix.target" +fi + +# We shouldn't reach this statement; error if we do. +exit 1 diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 840ce21e..cca459f0 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -128,14 +128,26 @@ function upgrade_in_place() { # [[ "$DLPX_UPGRADE_DRY_RUN" == "true" ]] && return + CURRENT_VERSION=$(get_current_version) || die "failed to get version" + + source_version_information + set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" + set_upgrade_property "UPGRADE_VERSION" "$VERSION" || + die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" + + if [[ -n "$HOTFIX" ]]; then + set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || + die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" + fi + set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || die "failed to set upgrade property 'UPGRADE_BASE_CONTAINER'" - set_upgrade_property "UPGRADE_BASE_VERSION" "$(get_current_version)" || + set_upgrade_property "UPGRADE_BASE_VERSION" "$CURRENT_VERSION" || die "failed to set upgrade property 'UPGRADE_BASE_VERSION'" # @@ -147,42 +159,13 @@ function upgrade_in_place() { cleanup_in_place_upgrade trap - EXIT - ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" - [[ -n "$ROOTFS_CONTAINER" ]] || - die "unable to determine currently mounted rootfs container" - - [[ -f "/var/lib/delphix-appliance/platform" ]] || - die "could not determine platform; file does not exist" - - "$IMAGE_PATH/execute" \ - -p "$(cat /var/lib/delphix-appliance/platform)" || - die "'$IMAGE_PATH/execute' failed in running appliance." - - # - # We use a seperate ZFS dataset for GRUB, and this dataset is - # generally not mounted when we update pacakges on the system. - # Thus, when a new kernel package is installed, via the call to - # "execute" above, the GRUB configuration will not be modified - # to use that new kernel. - # - # In order for the system to use the new kernel after a reboot, - # we must regenerate the GRUB configuration after the new kernel - # has been installed. The "rootfs-container set-bootfs" command - # will do just that; it knows how to mount our GRUB specific - # dataset, and how properly update the GRUB configuration. - # - "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || - die "failed to set-bootfs '$ROOTFS_CONTAINER'" - + local opt_f="" if [[ "$UPGRADE_TYPE" == "FULL" ]]; then - systemctl reboot || die "'systemctl reboot' failed" - else - systemctl reload delphix-platform || - die "'systemctl reload delphix-platform' failed" - - systemctl restart delphix-platform || - die "'systemctl restart delphix-platform' failed" + opt_f="-f" fi + + "$IMAGE_PATH/execute" "$opt_f" || + die "'$IMAGE_PATH/execute' failed in running appliance." } function cleanup_not_in_place_upgrade() { @@ -269,8 +252,18 @@ function upgrade_not_in_place() { # trap - EXIT - set_upgrade_property "UPGRADE_TYPE" "FULL" || - die "failed to set upgrade property 'UPGRADE_TYPE'" + source_version_information + + set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || + die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" + + set_upgrade_property "UPGRADE_VERSION" "$VERSION" || + die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" + + if [[ -n "$HOTFIX" ]]; then + set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || + die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" + fi set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index e130a0d8..1b4fec54 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -813,18 +813,8 @@ function migrate_configuration() { } function do_upgrade_container_in_place() { - [[ -f "/var/lib/delphix-appliance/platform" ]] || - die "could not determine platform; file does not exist" - - run "$IMAGE_PATH/execute" \ - -p "$(cat /var/lib/delphix-appliance/platform)" || + run "$IMAGE_PATH/execute" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" - - run /bin/systemctl reload delphix-platform || - die "'systemctl reload delphix-platform' failed in '$CONTAINER'" - - run /bin/systemctl restart delphix-platform || - die "'systemctl restart delphix-platform' failed in '$CONTAINER'" } function do_upgrade_container_not_in_place() { @@ -835,9 +825,6 @@ function do_upgrade_container_not_in_place() { -p "$(cat /var/lib/delphix-appliance/platform)" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" - run /bin/systemctl start delphix-platform || - die "'systemctl start delphix-platform' failed in '$CONTAINER'" - migrate_configuration || die "failed to migrate configuration for '$CONTAINER'" } From b1ffadcb1285002b97966b61b334b253f07870fd Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 6 Jun 2023 14:10:29 -0700 Subject: [PATCH 181/197] DLPX-86405 deferred upgrade via scripts fails (#727) --- upgrade/upgrade-scripts/upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index cca459f0..a8781b26 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -164,7 +164,7 @@ function upgrade_in_place() { opt_f="-f" fi - "$IMAGE_PATH/execute" "$opt_f" || + "$IMAGE_PATH/execute" $opt_f || die "'$IMAGE_PATH/execute' failed in running appliance." } From ae0ef38b60af77e9edecdf02bf54c57042370da4 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 13 Jun 2023 09:04:53 -0700 Subject: [PATCH 182/197] DLPX-86517 AWS 6.0.15.0(min) to 12.0.0.0 release upgrade apply failed. (#728) --- upgrade/upgrade-scripts/execute | 130 +++------------------- upgrade/upgrade-scripts/upgrade | 67 ++++++----- upgrade/upgrade-scripts/upgrade-container | 15 ++- 3 files changed, 68 insertions(+), 144 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 06eb078c..afc38481 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -24,41 +24,10 @@ set -o pipefail function usage() { echo "$(basename "$0"): $*" >&2 - echo "Usage: $(basename "$0") [-f] [-p ]" + echo "Usage: $(basename "$0") -p " exit 2 } -function post_alert() { - local jmxtool="/opt/delphix/server/bin/jmxtool" - - # - # Alerts are a virtualization service concept. Thus, if we're - # running on a variant that doesn't have the virtualization - # package installed, skip the alert. - # - [[ ! -x "$jmxtool" ]] && return - - # - # Skip the alert when running in an upgrade container, as the - # alert is only meant to notify the user about the host. - # - systemd-detect-virt -qc && return - - # - # The alert isn't critical, so if it fails to post, that's - # acceptiable. Further, it's possible to execute this script - # without the virtualization service running. As a result, we - # use "-w" and don't worry if it returns an error code. - # - if [[ "$1" == "reboot" ]]; then - $jmxtool -w boot upgrade server &>/dev/null - elif [[ "$1" == "restart" ]]; then - $jmxtool -w boot upgrade management &>/dev/null - else - die "invalid alert specified: '$1'" - fi -} - function generate_interface_to_mac_address_map() { INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" @@ -78,40 +47,17 @@ function generate_interface_to_mac_address_map() { die "Failed to generate map from interface names to mac addresses" } -# -# Specifies the platform to upgrade to; by default choose the same -# platform the script is running on. -# -# For not-in-place upgrades, we cannot use the get-appliance-platform -# script to determine the platform, hence why this option exists. -# -# This option should not be used to change platforms via an upgrade. -# -opt_p="" - -# -# Perform a "full" upgrade, which does a system reboot. By default, we -# perform a "deferred" upgrade, which resetarts services, but does not -# reboot the system. -# -opt_f="" - -while getopts ':fp:' c; do +while getopts ':rlBfsp:' c; do case $c in - f) eval "opt_$c=true" ;; - p) eval "opt_$c='$OPTARG'" ;; + r | l | B | f | s) ;; # LX-72: For now, silently ignore these. + p) + platform=$OPTARG + ;; *) usage "illegal options -- $OPTARG" ;; esac done -shift $((OPTIND - 1)) - -[[ $# -ne 0 ]] && usage "too many arguments specified" -[[ "$EUID" -ne 0 ]] && die "must be run as root" -if [[ -z "$opt_p" ]]; then - opt_p="$(get-appliance-platform)" || - usage "platform must be specified" -fi +[[ -z "$platform" ]] && usage "platform must be specified" # # When upgrading the packages on with this script, we want to ensure @@ -259,8 +205,8 @@ apt_get update || die "failed to update apt sources" # by installing delphix-virtualization at a later stage of the build via # ansible hooks, when the delphix-platform package has already been installed. # -if ! dpkg-query -l "delphix-platform-$opt_p" &>/dev/null; then - apt_get install -y "delphix-platform-$opt_p" || +if ! dpkg-query -l "delphix-platform-$platform" &>/dev/null; then + apt_get install -y "delphix-platform-$platform" || die "failed to install delphix-platform" fi @@ -324,7 +270,7 @@ dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || # shellcheck disable=SC2153 apt_get install \ -y --allow-downgrades --reinstall \ - "delphix-entire-$opt_p=$VERSION" || + "delphix-entire-$platform=$VERSION" || die "upgrade failed; from '$CURRENT_VERSION' to '$VERSION'" # @@ -336,17 +282,17 @@ apt_get install \ # particularly in the case of "--reinstall", which replaces the current # package with a new package of the same version (i.e. for hotfixes). # -apt-mark manual "delphix-entire-$opt_p" || +apt-mark manual "delphix-entire-$platform" || die "failed to mark 'delphix-entire' package as 'manual' installed" -[[ -f "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" ]] || +[[ -f "/usr/share/doc/delphix-entire-$platform/packages.list.gz" ]] || die "delphix-entire's packages.list.gz file is missing" -zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | xargs_apt_get install -y --allow-downgrades || die "failed to install packages listed in packages.list.gz file" -zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | cut -d= -f1 | xargs apt-mark manual || die "failed to mark as manual packages listed in packages.list.gz file" @@ -448,7 +394,7 @@ stop_stderr_redirect_to_system_log # we verify the package is installed and its version is correct; this # is simply to help us be confident that upgrade behaves as we expect. # -zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | sed 's/=/ /' | +zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | sed 's/=/ /' | while read -r name version; do installed=$(dpkg-query -Wf '${Version}' "$name") compare_versions "$installed" "=" "$version" || @@ -551,56 +497,14 @@ fi # The container does not have visibility into the complete network namespace of the # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map -if ! systemd-detect-virt -qc && [[ "$opt_p" == "aws" ]]; then +if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then generate_interface_to_mac_address_map fi -# -# We use a seperate ZFS dataset for GRUB, and this dataset is generally -# not mounted when we update pacakges on the system. Thus, when a new -# kernel package is installed, via the call to "execute" above, the GRUB -# configuration will not be modified to use that new kernel. -# -# In order for the system to use the new kernel after a reboot, we must -# regenerate the GRUB configuration after the new kernel has been -# installed. The "rootfs-container set-bootfs" command will do just -# that; it knows how to mount our GRUB specific dataset, and how -# properly update the GRUB configuration. -# -# Note, we only want to update GRUB when running outside of an upgrade -# container; since executing an upgrade within an upgrade container -# should not affect the host system. -# -if ! systemd-detect-virt -qc; then - ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" - [[ -n "$ROOTFS_CONTAINER" ]] || - die "unable to determine currently mounted rootfs container" - - "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || - die "failed to set-bootfs '$ROOTFS_CONTAINER'" -fi - -systemctl reload delphix-platform.service || - die "failed to reload delphix-platform.service" - # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. # zpool sync rpool || die "'zpool sync rpool' failed" -if [[ -f "$UPDATE_DIR/upgrade.properties" ]]; then - source_upgrade_properties -fi - -if [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; then - post_alert "reboot" - exec systemctl reboot || die "failed to reboot" -else - post_alert "restart" - exec systemctl restart delphix.target || - die "failed to restart delphix.target" -fi - -# We shouldn't reach this statement; error if we do. -exit 1 +exit 0 diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index a8781b26..840ce21e 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -128,26 +128,14 @@ function upgrade_in_place() { # [[ "$DLPX_UPGRADE_DRY_RUN" == "true" ]] && return - CURRENT_VERSION=$(get_current_version) || die "failed to get version" - - source_version_information - set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" - set_upgrade_property "UPGRADE_VERSION" "$VERSION" || - die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" - - if [[ -n "$HOTFIX" ]]; then - set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || - die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" - fi - set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || die "failed to set upgrade property 'UPGRADE_BASE_CONTAINER'" - set_upgrade_property "UPGRADE_BASE_VERSION" "$CURRENT_VERSION" || + set_upgrade_property "UPGRADE_BASE_VERSION" "$(get_current_version)" || die "failed to set upgrade property 'UPGRADE_BASE_VERSION'" # @@ -159,13 +147,42 @@ function upgrade_in_place() { cleanup_in_place_upgrade trap - EXIT - local opt_f="" - if [[ "$UPGRADE_TYPE" == "FULL" ]]; then - opt_f="-f" - fi + ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" + [[ -n "$ROOTFS_CONTAINER" ]] || + die "unable to determine currently mounted rootfs container" + + [[ -f "/var/lib/delphix-appliance/platform" ]] || + die "could not determine platform; file does not exist" - "$IMAGE_PATH/execute" $opt_f || + "$IMAGE_PATH/execute" \ + -p "$(cat /var/lib/delphix-appliance/platform)" || die "'$IMAGE_PATH/execute' failed in running appliance." + + # + # We use a seperate ZFS dataset for GRUB, and this dataset is + # generally not mounted when we update pacakges on the system. + # Thus, when a new kernel package is installed, via the call to + # "execute" above, the GRUB configuration will not be modified + # to use that new kernel. + # + # In order for the system to use the new kernel after a reboot, + # we must regenerate the GRUB configuration after the new kernel + # has been installed. The "rootfs-container set-bootfs" command + # will do just that; it knows how to mount our GRUB specific + # dataset, and how properly update the GRUB configuration. + # + "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || + die "failed to set-bootfs '$ROOTFS_CONTAINER'" + + if [[ "$UPGRADE_TYPE" == "FULL" ]]; then + systemctl reboot || die "'systemctl reboot' failed" + else + systemctl reload delphix-platform || + die "'systemctl reload delphix-platform' failed" + + systemctl restart delphix-platform || + die "'systemctl restart delphix-platform' failed" + fi } function cleanup_not_in_place_upgrade() { @@ -252,18 +269,8 @@ function upgrade_not_in_place() { # trap - EXIT - source_version_information - - set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || - die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" - - set_upgrade_property "UPGRADE_VERSION" "$VERSION" || - die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" - - if [[ -n "$HOTFIX" ]]; then - set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || - die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" - fi + set_upgrade_property "UPGRADE_TYPE" "FULL" || + die "failed to set upgrade property 'UPGRADE_TYPE'" set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 1b4fec54..e130a0d8 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -813,8 +813,18 @@ function migrate_configuration() { } function do_upgrade_container_in_place() { - run "$IMAGE_PATH/execute" || + [[ -f "/var/lib/delphix-appliance/platform" ]] || + die "could not determine platform; file does not exist" + + run "$IMAGE_PATH/execute" \ + -p "$(cat /var/lib/delphix-appliance/platform)" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" + + run /bin/systemctl reload delphix-platform || + die "'systemctl reload delphix-platform' failed in '$CONTAINER'" + + run /bin/systemctl restart delphix-platform || + die "'systemctl restart delphix-platform' failed in '$CONTAINER'" } function do_upgrade_container_not_in_place() { @@ -825,6 +835,9 @@ function do_upgrade_container_not_in_place() { -p "$(cat /var/lib/delphix-appliance/platform)" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" + run /bin/systemctl start delphix-platform || + die "'systemctl start delphix-platform' failed in '$CONTAINER'" + migrate_configuration || die "failed to migrate configuration for '$CONTAINER'" } From 48a7358420a0d0fd89f0da3334fe26c9a35b15c9 Mon Sep 17 00:00:00 2001 From: Palash Gandhi <87093175+pgandhi-delphix@users.noreply.github.com> Date: Thu, 22 Jun 2023 09:30:03 -0700 Subject: [PATCH 183/197] DLPX-85904 Remove hardcoded git branch from appliance-build (#726) PR URL: https://www.github.com/delphix/appliance-build/pull/726 --- branch.config | 13 --------- scripts/aptly-repo-from-debs.sh | 4 +-- scripts/build-ancillary-repository.sh | 28 +------------------ scripts/common.sh | 40 ++++++++++++++++++++++----- scripts/run-live-build.sh | 39 +++++++++++--------------- 5 files changed, 51 insertions(+), 73 deletions(-) delete mode 100644 branch.config diff --git a/branch.config b/branch.config deleted file mode 100644 index 714ae614..00000000 --- a/branch.config +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2019 by Delphix. All rights reserved. -# - -# -# The "BRANCH" parameter tracks the upstream branch of appliance-build. It is -# used to determine which branch of the linux package mirror will be used for -# the build if UPSTREAM_PRODUCT_BRANCH is not set. UPSTREAM_PRODUCT_BRANCH is -# set when appliance build is built by the appliance-build Jenkins job. The -# UPSTREAM_BRANCH parameter should be updated by the release lead on branching -# - -UPSTREAM_BRANCH="develop" diff --git a/scripts/aptly-repo-from-debs.sh b/scripts/aptly-repo-from-debs.sh index a2ea12ca..4141545d 100755 --- a/scripts/aptly-repo-from-debs.sh +++ b/scripts/aptly-repo-from-debs.sh @@ -64,9 +64,7 @@ done # Download the delphix upgrade verification debian package, stored in the # combined-packages bundle. # -AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ - "$AWS_S3_URI_COMBINED_PACKAGES" \ - "linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") +AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri "$AWS_S3_URI_COMBINED_PACKAGES") WORK_DIRECTORY=$(mktemp -d -p "$TOP/upgrade" tmp.pkgs.XXXXXXXXXX) diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index 95e4d614..0bbae145 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -61,29 +61,6 @@ function build_ancillary_repository() { EOF } -# -# Set UPSTREAM_BRANCH. This will determine which version of the linux package -# mirror is used. -# -if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then - echo "UPSTREAM_PRODUCT_BRANCH is not set." - if ! source "$TOP/branch.config" 2>/dev/null; then - echo "No branch.config file found in repo root." - exit 1 - fi - - if [[ -z "$UPSTREAM_BRANCH" ]]; then - echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ - "Ensure branch.config is properly formatted with e.g." \ - "UPSTREAM_BRANCH=\"\"" - exit 1 - fi - echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." -else - UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" -fi -echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" - # # The packages produced by Delphix are stored in Amazon S3. # Thus, in order to populate the ancillary repository with these @@ -95,10 +72,7 @@ echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" # packages from there, otherwise determine the latest combined-packages URI # automatically. # - -AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri \ - "$AWS_S3_URI_COMBINED_PACKAGES" \ - "linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest") +AWS_S3_URI_COMBINED_PACKAGES=$(resolve_s3_uri "$AWS_S3_URI_COMBINED_PACKAGES") mkdir -p "$TOP/build" WORK_DIRECTORY=$(mktemp -d -p "$TOP/build" tmp.pkgs.XXXXXXXXXX) diff --git a/scripts/common.sh b/scripts/common.sh index 99f0942c..7efd2fef 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -25,22 +25,25 @@ function die() { function resolve_s3_uri() { local pkg_uri="$1" - local latest_subprefix="$2" - local bucket="snapshot-de-images" - local jenkinsid="jenkins-ops" local resolved_uri if [[ -n "$pkg_uri" ]]; then resolved_uri="$pkg_uri" - elif [[ -n "$latest_subprefix" ]]; then + else + # + # Set UPSTREAM_BRANCH. This will determine which version of the linux package + # mirror is used. + # + UPSTREAM_BRANCH=$(get_upstream_or_fail_if_unset) || exit 1 + echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + local latest_subprefix="linux-pkg/${UPSTREAM_BRANCH}/combine-packages/post-push/latest" + local bucket="snapshot-de-images" + local jenkinsid="jenkins-ops" aws s3 cp --quiet \ "s3://$bucket/builds/$jenkinsid/$latest_subprefix" . resolved_uri="s3://$bucket/$(cat latest)" rm -f latest - else - echo "Invalid arguments provided to resolve_s3_uri()" 2>&1 - exit 1 fi if aws s3 ls "$resolved_uri" &>/dev/null; then @@ -135,3 +138,26 @@ function extract_debs_into_dir() { find "$source_dir" -name '*.deb' -exec mv {} "$target_dir" \; find "$source_dir" -name '*.ddeb' -exec mv {} "$target_dir" \; } + +function get_upstream_or_fail_if_unset() { + if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then + local upstream_branch + upstream_branch="$(git rev-parse --abbrev-ref --symbolic-full-name "@{u}" | cut -d'/' -f2-)" + if [[ -z $upstream_branch ]]; then + echo "ERROR: The currently checked out branch" >&2 + echo " does not have an upstream branch configured. Set the" >&2 + echo " upstream branch you plan to push to:" >&2 + echo "" >&2 + echo " git branch --set-upstream-to=" >&2 + echo "" >&2 + echo " Then run this script again. '' can be " >&2 + echo " something like '6.0/stage'" >&2 + return 1 + else + echo "$upstream_branch" + return 0 + fi + else + echo "$UPSTREAM_PRODUCT_BRANCH" + fi +} diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index dd7a9bf3..2e5943eb 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -15,6 +15,8 @@ # limitations under the License. # +. "${BASH_SOURCE%/*}/common.sh" + TOP=$(git rev-parse --show-toplevel 2>/dev/null) if [[ -z "$TOP" ]]; then @@ -138,33 +140,17 @@ while ! curl --output /dev/null --silent --head --fail \ sleep 1 done -# -# Set UPSTREAM_BRANCH. This will determine which version of the linux package -# mirror is used. -# -if [[ -z "$UPSTREAM_PRODUCT_BRANCH" ]]; then - echo "UPSTREAM_PRODUCT_BRANCH is not set." - if ! source "$TOP/branch.config" 2>/dev/null; then - echo "No branch.config file found in repo root." - exit 1 - fi - - if [[ -z "$UPSTREAM_BRANCH" ]]; then - echo "UPSTREAM_BRANCH parameter was not sourced from branch.config." \ - "Ensure branch.config is properly formatted with e.g." \ - "UPSTREAM_BRANCH=\"\"" - exit 1 - fi - echo "Defaulting to branch $UPSTREAM_BRANCH set in branch.config." -else - UPSTREAM_BRANCH="$UPSTREAM_PRODUCT_BRANCH" -fi -echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" - pkg_mirror_secondary='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_SECONDARY" ]]; then pkg_mirror_secondary="$DELPHIX_PACKAGE_MIRROR_SECONDARY" else + # + # Set UPSTREAM_BRANCH. This will determine which version of the linux package + # mirror is used. + # + UPSTREAM_BRANCH=$(get_upstream_or_fail_if_unset) || exit 1 + echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + # # If no secondary package mirror is provided, then pull in the latest # mirror dataset for the build. If no latest dataset is found, then fail. @@ -188,6 +174,13 @@ pkg_mirror_main='' if [[ -n "$DELPHIX_PACKAGE_MIRROR_MAIN" ]]; then pkg_mirror_main="$DELPHIX_PACKAGE_MIRROR_MAIN" else + # + # Set UPSTREAM_BRANCH. This will determine which version of the linux package + # mirror is used. + # + UPSTREAM_BRANCH=$(get_upstream_or_fail_if_unset) || exit 1 + echo "Running with UPSTREAM_BRANCH set to ${UPSTREAM_BRANCH}" + # # If no main package mirror is provided, then pull in the latest mirror # dataset for the build. If no latest dataset is found, then fail. From c70783855faec8a2761db72f90f9a6d02db48e1c Mon Sep 17 00:00:00 2001 From: Palash Gandhi <87093175+palash-delphix@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:16:49 -0700 Subject: [PATCH 184/197] DLPX-86854 Post-upgrade cleanup task fails with internal error, due to attempting to delete dataset which has already been deleted (#730) PR URL: https://www.github.com/delphix/appliance-build/pull/730 --- upgrade/upgrade-scripts/rootfs-cleanup | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/upgrade/upgrade-scripts/rootfs-cleanup b/upgrade/upgrade-scripts/rootfs-cleanup index e2901241..e221c239 100755 --- a/upgrade/upgrade-scripts/rootfs-cleanup +++ b/upgrade/upgrade-scripts/rootfs-cleanup @@ -26,6 +26,10 @@ def version(rootfs: str) -> str: ]).decode("UTF-8").strip()) +def exists(rootfs: str) -> bool: + return not bool(subprocess.run(["zfs", "list", rootfs]).returncode) + + # # Helper class used when sorting rootfs filesystems and snapshots. This allows # us to use the "sorted" builtin, to build a list of all rootfs filesystems and @@ -111,6 +115,13 @@ def main() -> None: # after an upgrade (i.e. we never upgrade to a lower version). # for rootfs in sorted(filesystems + snapshots, key=rootfscmp)[:-2]: + # + # Skip if the snapshot or filesystem was destroyed by a previous + # iteration. + # + if not exists(rootfs): + continue + # # In the event of a rollback, we want to be careful to not # delete the currently running version, as well as any versions From c30d05daf9326d4568d05a15baaef087378dc44a Mon Sep 17 00:00:00 2001 From: Palash Gandhi <87093175+palash-delphix@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:55:14 -0700 Subject: [PATCH 185/197] DLPX-87038 VMDK file size differs from the size in the OVF file (#732) PR URL: https://www.github.com/delphix/appliance-build/pull/732 --- .../config/hooks/vm-artifacts/92-ova-machine-image.binary | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/live-build/config/hooks/vm-artifacts/92-ova-machine-image.binary b/live-build/config/hooks/vm-artifacts/92-ova-machine-image.binary index f7d54564..582d2c70 100755 --- a/live-build/config/hooks/vm-artifacts/92-ova-machine-image.binary +++ b/live-build/config/hooks/vm-artifacts/92-ova-machine-image.binary @@ -47,8 +47,7 @@ esac DATE=$(date --utc --iso-8601=seconds) -VMDK_FILESIZE=$(qemu-img info --output=json "$ARTIFACT_NAME.vmdk" | - jq '.["actual-size"]') +VMDK_FILESIZE=$(stat -c %s "$ARTIFACT_NAME.vmdk") VMDK_CAPACITY=$(qemu-img info --output=json "$ARTIFACT_NAME.vmdk" | jq '.["virtual-size"]') From 2171c93578b6a808cf927067e9225e2d6b4901f7 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 16 Aug 2023 09:51:54 -0700 Subject: [PATCH 186/197] TOOL-22136 Update README.md with new buildserver image name (#734) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1574bf69..40e60785 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ EC2, Microsoft Azure, VMware, OpenStack). Run this command on "dlpxdc.co" to create the VM used to do the build: - $ dc clone-latest --size COMPUTE_LARGE bootstrap-20-04 $USER-bootstrap + $ dc clone-latest --size COMPUTE_LARGE dlpx-internal-buildserver-develop $USER-bootstrap -Log into that VM using the "ubuntu" user, and run these commands: +Log into that VM using the "delphix" user, and run these commands: $ git clone https://github.com/delphix/appliance-build.git $ cd appliance-build From d59753489786c2ab946d7144cfb6dcc5cc59240e Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Wed, 6 Sep 2023 14:26:24 -0700 Subject: [PATCH 187/197] DLPX-87792 Fix timeout when starting an upgrade container (#737) --- upgrade/upgrade-scripts/upgrade-container | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index e130a0d8..6a0a2196 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -462,6 +462,7 @@ function start() { die "container '$CONTAINER' non-existent or mis-configured" fi + systemctl restart systemd-reexec.service 2>/dev/null machinectl start "$CONTAINER" || die "failed to start container '$CONTAINER'" @@ -486,6 +487,7 @@ function start() { } function stop() { + systemctl restart systemd-reexec.service 2>/dev/null machinectl terminate "$CONTAINER" || die "failed to termiante container: '$CONTAINER'" From 0bdff1c7e3c28453c594dbdc8117e29e2c65bb86 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Mon, 18 Sep 2023 10:56:43 -0700 Subject: [PATCH 188/197] TOOL-22969 Add mold to development engines (#738) --- .../tasks/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index 83f22199..2df342f7 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -45,6 +45,7 @@ - llvm-12 - lsscsi - mandoc + - mold - nfs-kernel-server - parted - pkg-config @@ -76,3 +77,14 @@ group: staff state: directory recurse: yes + +- file: + path: "/export/home/delphix/.cargo/" + state: directory + owner: delphix + group: staff +- copy: + dest: "/export/home/delphix/.cargo/config.toml" + content: | + [target.x86_64-unknown-linux-gnu] + rustflags = ["-C", "link-arg=-B/usr/libexec/mold"] From 118ab7f85242051e605621fe4a3748077495723e Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Wed, 4 Oct 2023 08:01:10 -0700 Subject: [PATCH 189/197] DLPX-88183 Generate BTF data for ZFS kernel module during git-zfs-make (#740) PR URL: https://www.github.com/delphix/appliance-build/pull/740 --- .../appliance-build.minimal-common/tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index 9bb70ec9..d95545e6 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -69,3 +69,18 @@ delay: 30 register: result until: result is succeeded + +- shell: ls /lib/modules + register: kernel_versions + +# +# In order to generate BTF data when building the ZFS kernel module +# through git-zfs-make we need to provide a symlink of our vmlinux +# (with debug info) in the kernel header source directory which is +# referenced during the kernel module's compilation. +# +- ansible.builtin.file: + src: '/usr/lib/debug/boot/vmlinux-{{ item }}' + dest: '/usr/src/linux-headers-{{ item }}/vmlinux' + state: link + loop: '{{ kernel_versions.stdout_lines }}' From a9f8eeab6e004b931e1a11d016eaec7d20128919 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 9 Oct 2023 13:48:20 -0700 Subject: [PATCH 190/197] DLPX-85893 run upgrade "execute" script from separate service (#731) --- upgrade/upgrade-scripts/execute | 130 +++++++++++++++++++--- upgrade/upgrade-scripts/upgrade | 67 +++++------ upgrade/upgrade-scripts/upgrade-container | 15 +-- 3 files changed, 144 insertions(+), 68 deletions(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index afc38481..06eb078c 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -24,10 +24,41 @@ set -o pipefail function usage() { echo "$(basename "$0"): $*" >&2 - echo "Usage: $(basename "$0") -p " + echo "Usage: $(basename "$0") [-f] [-p ]" exit 2 } +function post_alert() { + local jmxtool="/opt/delphix/server/bin/jmxtool" + + # + # Alerts are a virtualization service concept. Thus, if we're + # running on a variant that doesn't have the virtualization + # package installed, skip the alert. + # + [[ ! -x "$jmxtool" ]] && return + + # + # Skip the alert when running in an upgrade container, as the + # alert is only meant to notify the user about the host. + # + systemd-detect-virt -qc && return + + # + # The alert isn't critical, so if it fails to post, that's + # acceptiable. Further, it's possible to execute this script + # without the virtualization service running. As a result, we + # use "-w" and don't worry if it returns an error code. + # + if [[ "$1" == "reboot" ]]; then + $jmxtool -w boot upgrade server &>/dev/null + elif [[ "$1" == "restart" ]]; then + $jmxtool -w boot upgrade management &>/dev/null + else + die "invalid alert specified: '$1'" + fi +} + function generate_interface_to_mac_address_map() { INTERFACE_TO_MACADRESS_MAP_FILE_PATH="/etc/interface_to_macaddress_map.out" @@ -47,17 +78,40 @@ function generate_interface_to_mac_address_map() { die "Failed to generate map from interface names to mac addresses" } -while getopts ':rlBfsp:' c; do +# +# Specifies the platform to upgrade to; by default choose the same +# platform the script is running on. +# +# For not-in-place upgrades, we cannot use the get-appliance-platform +# script to determine the platform, hence why this option exists. +# +# This option should not be used to change platforms via an upgrade. +# +opt_p="" + +# +# Perform a "full" upgrade, which does a system reboot. By default, we +# perform a "deferred" upgrade, which resetarts services, but does not +# reboot the system. +# +opt_f="" + +while getopts ':fp:' c; do case $c in - r | l | B | f | s) ;; # LX-72: For now, silently ignore these. - p) - platform=$OPTARG - ;; + f) eval "opt_$c=true" ;; + p) eval "opt_$c='$OPTARG'" ;; *) usage "illegal options -- $OPTARG" ;; esac done +shift $((OPTIND - 1)) + +[[ $# -ne 0 ]] && usage "too many arguments specified" +[[ "$EUID" -ne 0 ]] && die "must be run as root" -[[ -z "$platform" ]] && usage "platform must be specified" +if [[ -z "$opt_p" ]]; then + opt_p="$(get-appliance-platform)" || + usage "platform must be specified" +fi # # When upgrading the packages on with this script, we want to ensure @@ -205,8 +259,8 @@ apt_get update || die "failed to update apt sources" # by installing delphix-virtualization at a later stage of the build via # ansible hooks, when the delphix-platform package has already been installed. # -if ! dpkg-query -l "delphix-platform-$platform" &>/dev/null; then - apt_get install -y "delphix-platform-$platform" || +if ! dpkg-query -l "delphix-platform-$opt_p" &>/dev/null; then + apt_get install -y "delphix-platform-$opt_p" || die "failed to install delphix-platform" fi @@ -270,7 +324,7 @@ dpkg-query -Wf '${Package}\n' | xargs apt-mark auto || # shellcheck disable=SC2153 apt_get install \ -y --allow-downgrades --reinstall \ - "delphix-entire-$platform=$VERSION" || + "delphix-entire-$opt_p=$VERSION" || die "upgrade failed; from '$CURRENT_VERSION' to '$VERSION'" # @@ -282,17 +336,17 @@ apt_get install \ # particularly in the case of "--reinstall", which replaces the current # package with a new package of the same version (i.e. for hotfixes). # -apt-mark manual "delphix-entire-$platform" || +apt-mark manual "delphix-entire-$opt_p" || die "failed to mark 'delphix-entire' package as 'manual' installed" -[[ -f "/usr/share/doc/delphix-entire-$platform/packages.list.gz" ]] || +[[ -f "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" ]] || die "delphix-entire's packages.list.gz file is missing" -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | xargs_apt_get install -y --allow-downgrades || die "failed to install packages listed in packages.list.gz file" -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | cut -d= -f1 | xargs apt-mark manual || die "failed to mark as manual packages listed in packages.list.gz file" @@ -394,7 +448,7 @@ stop_stderr_redirect_to_system_log # we verify the package is installed and its version is correct; this # is simply to help us be confident that upgrade behaves as we expect. # -zcat "/usr/share/doc/delphix-entire-$platform/packages.list.gz" | sed 's/=/ /' | +zcat "/usr/share/doc/delphix-entire-$opt_p/packages.list.gz" | sed 's/=/ /' | while read -r name version; do installed=$(dpkg-query -Wf '${Version}' "$name") compare_versions "$installed" "=" "$version" || @@ -497,14 +551,56 @@ fi # The container does not have visibility into the complete network namespace of the # engine. Hence we need to create the map outside the container to ensure the presense # of all the interfaces in the map -if ! systemd-detect-virt --container --quiet && [[ $(get-appliance-platform) == "aws" ]]; then +if ! systemd-detect-virt -qc && [[ "$opt_p" == "aws" ]]; then generate_interface_to_mac_address_map fi +# +# We use a seperate ZFS dataset for GRUB, and this dataset is generally +# not mounted when we update pacakges on the system. Thus, when a new +# kernel package is installed, via the call to "execute" above, the GRUB +# configuration will not be modified to use that new kernel. +# +# In order for the system to use the new kernel after a reboot, we must +# regenerate the GRUB configuration after the new kernel has been +# installed. The "rootfs-container set-bootfs" command will do just +# that; it knows how to mount our GRUB specific dataset, and how +# properly update the GRUB configuration. +# +# Note, we only want to update GRUB when running outside of an upgrade +# container; since executing an upgrade within an upgrade container +# should not affect the host system. +# +if ! systemd-detect-virt -qc; then + ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" + [[ -n "$ROOTFS_CONTAINER" ]] || + die "unable to determine currently mounted rootfs container" + + "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || + die "failed to set-bootfs '$ROOTFS_CONTAINER'" +fi + +systemctl reload delphix-platform.service || + die "failed to reload delphix-platform.service" + # # Before we exit, we want to ensure all of the changes made to the root # pool via the packaging operations above have made it to disk. # zpool sync rpool || die "'zpool sync rpool' failed" -exit 0 +if [[ -f "$UPDATE_DIR/upgrade.properties" ]]; then + source_upgrade_properties +fi + +if [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; then + post_alert "reboot" + exec systemctl reboot || die "failed to reboot" +else + post_alert "restart" + exec systemctl restart delphix.target || + die "failed to restart delphix.target" +fi + +# We shouldn't reach this statement; error if we do. +exit 1 diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index 840ce21e..cca459f0 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -128,14 +128,26 @@ function upgrade_in_place() { # [[ "$DLPX_UPGRADE_DRY_RUN" == "true" ]] && return + CURRENT_VERSION=$(get_current_version) || die "failed to get version" + + source_version_information + set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" + set_upgrade_property "UPGRADE_VERSION" "$VERSION" || + die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" + + if [[ -n "$HOTFIX" ]]; then + set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || + die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" + fi + set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || die "failed to set upgrade property 'UPGRADE_BASE_CONTAINER'" - set_upgrade_property "UPGRADE_BASE_VERSION" "$(get_current_version)" || + set_upgrade_property "UPGRADE_BASE_VERSION" "$CURRENT_VERSION" || die "failed to set upgrade property 'UPGRADE_BASE_VERSION'" # @@ -147,42 +159,13 @@ function upgrade_in_place() { cleanup_in_place_upgrade trap - EXIT - ROOTFS_CONTAINER="$(get_mounted_rootfs_container_name)" - [[ -n "$ROOTFS_CONTAINER" ]] || - die "unable to determine currently mounted rootfs container" - - [[ -f "/var/lib/delphix-appliance/platform" ]] || - die "could not determine platform; file does not exist" - - "$IMAGE_PATH/execute" \ - -p "$(cat /var/lib/delphix-appliance/platform)" || - die "'$IMAGE_PATH/execute' failed in running appliance." - - # - # We use a seperate ZFS dataset for GRUB, and this dataset is - # generally not mounted when we update pacakges on the system. - # Thus, when a new kernel package is installed, via the call to - # "execute" above, the GRUB configuration will not be modified - # to use that new kernel. - # - # In order for the system to use the new kernel after a reboot, - # we must regenerate the GRUB configuration after the new kernel - # has been installed. The "rootfs-container set-bootfs" command - # will do just that; it knows how to mount our GRUB specific - # dataset, and how properly update the GRUB configuration. - # - "$IMAGE_PATH/rootfs-container" set-bootfs "$ROOTFS_CONTAINER" || - die "failed to set-bootfs '$ROOTFS_CONTAINER'" - + local opt_f="" if [[ "$UPGRADE_TYPE" == "FULL" ]]; then - systemctl reboot || die "'systemctl reboot' failed" - else - systemctl reload delphix-platform || - die "'systemctl reload delphix-platform' failed" - - systemctl restart delphix-platform || - die "'systemctl restart delphix-platform' failed" + opt_f="-f" fi + + "$IMAGE_PATH/execute" "$opt_f" || + die "'$IMAGE_PATH/execute' failed in running appliance." } function cleanup_not_in_place_upgrade() { @@ -269,8 +252,18 @@ function upgrade_not_in_place() { # trap - EXIT - set_upgrade_property "UPGRADE_TYPE" "FULL" || - die "failed to set upgrade property 'UPGRADE_TYPE'" + source_version_information + + set_upgrade_property "UPGRADE_TYPE" "$UPGRADE_TYPE" || + die "failed to set upgrade property 'UPGRADE_TYPE' to '$UPGRADE_TYPE'" + + set_upgrade_property "UPGRADE_VERSION" "$VERSION" || + die "failed to set upgrade property 'UPGRADE_VERSION' to '$VERSION'" + + if [[ -n "$HOTFIX" ]]; then + set_upgrade_property "UPGRADE_HOTFIX" "$HOTFIX" || + die "failed to set upgrade property 'UPGRADE_HOTFIX' to '$HOTFIX'" + fi set_upgrade_property "UPGRADE_BASE_CONTAINER" \ "$(get_mounted_rootfs_container_name)" || diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 6a0a2196..cc2558c3 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -815,18 +815,8 @@ function migrate_configuration() { } function do_upgrade_container_in_place() { - [[ -f "/var/lib/delphix-appliance/platform" ]] || - die "could not determine platform; file does not exist" - - run "$IMAGE_PATH/execute" \ - -p "$(cat /var/lib/delphix-appliance/platform)" || + run "$IMAGE_PATH/execute" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" - - run /bin/systemctl reload delphix-platform || - die "'systemctl reload delphix-platform' failed in '$CONTAINER'" - - run /bin/systemctl restart delphix-platform || - die "'systemctl restart delphix-platform' failed in '$CONTAINER'" } function do_upgrade_container_not_in_place() { @@ -837,9 +827,6 @@ function do_upgrade_container_not_in_place() { -p "$(cat /var/lib/delphix-appliance/platform)" || die "'$IMAGE_PATH/execute' failed in '$CONTAINER'" - run /bin/systemctl start delphix-platform || - die "'systemctl start delphix-platform' failed in '$CONTAINER'" - migrate_configuration || die "failed to migrate configuration for '$CONTAINER'" } From 8f571d5e001bd5dd4f539e0bfe92e9d8ebd7ba82 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 12 Oct 2023 10:08:50 -0700 Subject: [PATCH 191/197] DLPX-85893 run upgrade "execute" script from separate service (#741) --- upgrade/upgrade-scripts/upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/upgrade b/upgrade/upgrade-scripts/upgrade index cca459f0..a8781b26 100755 --- a/upgrade/upgrade-scripts/upgrade +++ b/upgrade/upgrade-scripts/upgrade @@ -164,7 +164,7 @@ function upgrade_in_place() { opt_f="-f" fi - "$IMAGE_PATH/execute" "$opt_f" || + "$IMAGE_PATH/execute" $opt_f || die "'$IMAGE_PATH/execute' failed in running appliance." } From b71c3cd6b7a8bf413bfffb2eebf0560fa1574910 Mon Sep 17 00:00:00 2001 From: Palash Gandhi <87093175+palash-delphix@users.noreply.github.com> Date: Wed, 18 Oct 2023 13:55:45 -0700 Subject: [PATCH 192/197] DLPX-84565 telegraf.service restarts after upgrade when it should be masked (#739) PR URL: https://www.github.com/delphix/appliance-build/pull/739 --- upgrade/upgrade-scripts/common.sh | 14 +++++++++++++- upgrade/upgrade-scripts/execute | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index c52058eb..4035b077 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -381,7 +381,7 @@ function mask_service() { chroot "/var/lib/machines/$container" systemctl mask "$svc" || die "failed to mask '$svc' in container '$container'" else - systemctl mask "$svc" || die "failed to mask '$svc'" + systemctl mask --now "$svc" || die "failed to mask '$svc'" fi } @@ -442,6 +442,17 @@ function fix_and_migrate_services() { fi fi + # + # Due to https://github.com/influxdata/telegraf/issues/14052, telegraf must be masked after + # packages are upgraded. The telegraf package removes /etc/systemd/system/telegraf.service thus + # reversing the `systemctl mask` operation performed before the packages are upgraded. + # Once this issue is fixed and a version with the fix makes it into the product, this can be + # removed. + # + if [[ "$(systemctl is-enabled telegraf)" == enabled ]]; then + mask_service telegraf "$container" + fi + # # The services listed below are either permanently disabled or can be # dynamically modified by the application(s) running on the appliance, @@ -483,5 +494,6 @@ function fix_and_migrate_services() { snmpd.service systemd-timesyncd.service td-agent.service + telegraf.service EOF } diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 06eb078c..25b0bcd7 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -434,6 +434,17 @@ dpkg-query -Wf '${Conffiles}\n' | awk '$3 == "obsolete" {print $1}' | die "failed to reinstall package '$package'" done || die "failed to remove obsolete package configuration files" +# +# Due to https://github.com/influxdata/telegraf/issues/14052, telegraf must be masked after +# packages are upgraded. The telegraf package removes /etc/systemd/system/telegraf.service thus +# reversing the `systemctl mask` operation performed before the packages are upgraded. +# Once this issue is fixed and a version with the fix makes it into the product, this can be +# removed. +# +if [[ "$(systemctl is-enabled telegraf)" == enabled ]]; then + systemctl mask --now telegraf.service +fi + # # Unmask docker, which was masked at the beginning of the upgrade due # to DLPX-77949. From 7cb8f10eb25d57474bee13d2c87e8a53b78eb49b Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 Nov 2023 11:23:23 -0800 Subject: [PATCH 193/197] DLPX-88546 upgrade verification failed to generate faults (#743) --- upgrade/upgrade-scripts/upgrade-container | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index cc2558c3..075c61d9 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -422,6 +422,21 @@ function create_upgrade_container() { die "failed to add '/domain0' to container config file" fi + # + # Lastly, we create the log directory and bind mount this into + # the container. This is meant as a way for software running in + # the container, to share files with software running on the + # host; e.g. during upgrade verify, this directory can be used + # to store logs, such that they persist after the container is + # destroyed. + # + mkdir -p "$LOG_DIRECTORY" || + die "failed to create directory: '$LOG_DIRECTORY'" + cat >>"/etc/systemd/nspawn/$CONTAINER.nspawn" <<-EOF || + Bind=$LOG_DIRECTORY + EOF + die "failed to add '$LOG_DIRECTORY' to container config file" + # # We want to enable all available capabilities to the container # that we will use to run the ugprade verification. Ideally, we From a1b7edf79189d0718907826518c3b21450b9bfa7 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 14 Nov 2023 11:14:43 -0800 Subject: [PATCH 194/197] DLPX-88573 Perform deferred upgrade prior to VDB downtime for FULL upgrade (#742) --- upgrade/upgrade-scripts/execute | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 25b0bcd7..b076d2eb 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -604,7 +604,13 @@ if [[ -f "$UPDATE_DIR/upgrade.properties" ]]; then source_upgrade_properties fi -if [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; then +# +# On versions 18.0 and greater, we don't issue the reboot. Rather, +# we restart the delphix services, and the reboot will be issued by +# the virtualization service as it starts up. +# +if { [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; } && + compare_versions "$CURRENT_VERSION" lt "18.0.0.0-0"; then post_alert "reboot" exec systemctl reboot || die "failed to reboot" else From a4ef28e3e1de9cd98c69ecaf28de40cc4a3aa5cf Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 8 Jan 2024 09:27:53 -0800 Subject: [PATCH 195/197] DLPX-89392 Race between reboot and grub install (#747) --- upgrade/upgrade-scripts/execute | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index b076d2eb..b374e24e 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -612,9 +612,26 @@ fi if { [[ -n "$opt_f" ]] || [[ "$UPGRADE_TYPE" == "FULL" ]]; } && compare_versions "$CURRENT_VERSION" lt "18.0.0.0-0"; then post_alert "reboot" - exec systemctl reboot || die "failed to reboot" + + # + # We don't want this script to return until after the reboot has + # been completed; i.e. don't exit. Since "systemctl reboot" will + # initiate a reboot, but return prior to the reboot having been + # completed, we rely on the sleep to prevent this script from + # exiting prior to the reboot. + # + systemctl reboot || die "failed to reboot" + exec sleep infinity || die "failed to wait for the reboot" else post_alert "restart" + + # + # Similar to the reboot case above, we don't want this script + # to return until after the delphix services have been restarted. + # Since "systemctl restart" will initiate the restart of the + # services, and wait for them to become active before it returns, + # we don't need to add a sleep like we did for the reboot case. + # exec systemctl restart delphix.target || die "failed to restart delphix.target" fi From 8c70362bfa4f037a0c4ed0dd6caafdaf23568764 Mon Sep 17 00:00:00 2001 From: Sanjipta Behera Date: Thu, 1 Feb 2024 15:14:07 +0530 Subject: [PATCH 196/197] TOOL-24633 Revert QI-3191 dlpx-qa-gate not installing on DCoLs as Jenkins agents PR URL: https://www.github.com/delphix/appliance-build/pull/751 --- .../variants/internal-dcenter/package-lists/dcenter.list.chroot | 1 - 1 file changed, 1 deletion(-) diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index 9c074cb6..3c92032f 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -37,4 +37,3 @@ python3-toml python3-venv targetcli-fb telnet -unixodbc-dev From e60d20a957ca3d14de8811e5726e030e58279c96 Mon Sep 17 00:00:00 2001 From: Jules Tamagnan <77420472+jtamagnan-delphix@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:12:14 -0800 Subject: [PATCH 197/197] TOOL-24608 Remove unused Python dependencies (#750) PR URL: https://www.github.com/delphix/appliance-build/pull/750 --- .../internal-dcenter/package-lists/dcenter.list.chroot | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot index 9c074cb6..b1f63e9c 100644 --- a/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot +++ b/live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot @@ -1,5 +1,5 @@ # -# Copyright 2021 Delphix +# Copyright 2021, 2024 Delphix # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,14 +26,6 @@ nfs-kernel-server openjdk-11-jdk-headless python3 python3-dev -python3-ldap -python3-marshmallow -python3-marshmallow-doc -python3-pip -python3-pyvmomi -python3-six -python3-tenacity -python3-toml python3-venv targetcli-fb telnet