diff --git a/README b/README index d4c0e9a..3a5936a 100644 --- a/README +++ b/README @@ -1,6 +1,24 @@ -Author: William Lam -Website: http://www.virtuallyghetto.com/ +# ghettoVCB -ghettoVCB Documentation - http://communities.vmware.com/docs/DOC-8760 -ghettoVCB VMTN Group - http://communities.vmware.com/groups/ghettovcb -ghettoVCB Restore Documentation - http://communities.vmware.com/docs/DOC-10595 +### Description + +The ghettoVCB script performs backups of virtual machines residing on ESX(i) 3.x, 4.x, 5.x & 6.x servers using methodology similar to VMware's VCB tool. The script takes snapshots of live running virtual machines, backs up the master VMDK(s) and then upon completion, deletes the snapshot until the next backup. The only caveat is that it utilizes resources available to the ESXi Shell running the backups as opposed to following the traditional method of offloading virtual machine backups through a VCB proxy. + +### How to install + +You can quickly install ghettoVCB by downloading and install either the VIB or offline bundle using the following commands: + +Install VIB +``` +esxcli software vib install -v /vghetto-ghettoVCB.vib -f +``` + +Install offline bundle +``` +esxcli software vib install -d /vghetto-ghettoVCB-offline-bundle.zip -f +``` + +### Additional Documentation & Resources +- [ghettoVCB Documentation](http://communities.vmware.com/docs/DOC-8760) +- [ghettoVCB VMTN Group](http://communities.vmware.com/groups/ghettovcb) +- [ghettoVCB Restore Documentation](http://communities.vmware.com/docs/DOC-10595) diff --git a/README.md b/README.md new file mode 100644 index 0000000..f5a2347 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +ghettoVCB +=== +This modified bundle is based on fork of lamw/ghettoVCB master branch. + + Author: William Lam + Website: http://www.virtuallyghetto.com/ + + ghettoVCB Documentation - http://communities.vmware.com/docs/DOC-8760 + ghettoVCB VMTN Group - http://communities.vmware.com/groups/ghettovcb + ghettoVCB Restore Documentation - http://communities.vmware.com/docs/DOC-10595 + +----------- +**NOTE!** + +> You will use this bundle in your environment at your own risk. The authors of this package can not be held responsible for any issues this bundle may cause in your system. + +================ + +History of modifications: +=== + +- 11.11.15 Merged modifications of lamw/ghettoVCB until 07.11.2015 into this bundle. Additional fixes and modifications listed below are included in this bundle. + +- 06.11.15 Merged modifications of lamw/ghettoVCB until 05.11.2015 into this bundle. + +- 23.10.14 Fixed value displayed as "DST_DATASTORE_FREE" in debug mode. Fixed message when faulty backup is deleted to match situation where nothing is deleted (VM_BACKUP_ROTATION_COUNT). Do not continue backup of vm if backup of vmdk's of it has failed for some reason. + +- 22.10.14 Fixed value displayed as "SRC_DATASTORE_FREE" in debug mode. Command'.. | grep -i "capacity" ..' returned two values in VMware 5.5.0: "maxVirtualDiskCapacity" and "capacity" => parameter -i removed. + +- 17.10.14 Fixed errors with optional pause: do not pause after last vmdk of vm. Added more debug messages. + +- 16.10.14 Fixed message "Succesfully removed lock directory" of debug mode. In case backup of any vmdk's of vm fails, make backup of vm fail too: do not compress faulty backup if compress is enabled (ENABLE_COMPRESSION), instead delete this faulty backup (in checkVMBackupRotation). Prev. is fix for symlink creation too because it has not been created in this case while backups were anyway rotated (RSYNC_LINK,SYMLINK_SRC). Changed location of compress and rsync blocks. Optional pause between backup of vmdk's (debug "file already exist" errors). + +- 01.10.14 Added VMKFSTOOLS_CMD_OPTIONS for VMKFSTOOLS_CMD command (f.ex VMKFSTOOLS_CMD_OPTIONS="-v 10") + +- 05.09.14 In ghettoVCB.sh fixed typo indepdenent => independent. + +- 29.08.14 Fixed handling of "-w" parameter, it does not have to be located before parameters -a and -m on command line anymore. + Trap settings for removing workdir modified in case WORKDIR_DEBUG=1, + also fixed call of reConfigureGhettoVCBConfiguration which was placed too late inside sanityCheck. + NOTE! Remember to use own ghettoVCB specific temp directory for WORKDIR, it may be removed in the end of script. + In ghettoVCB-restore.sh fixed one typo. + +- 27.08.14 Fixed status messages of dryrun. + Fixed "Final status" message when VM has independant VMDKs and at least one normal VMDK. + +- 23.08.14 Forked from https://github.com/lamw/ghettoVCB.git. + After help text is displayed, exit. + Spaces in the end of lines were removed by editor. Real modifications have been marked with dates. + + +======================== diff --git a/ghettoVCB-restore.sh b/ghettoVCB-restore.sh index 81673ea..f85100b 100755 --- a/ghettoVCB-restore.sh +++ b/ghettoVCB-restore.sh @@ -5,14 +5,14 @@ ###### DO NOT EDIT PASS THIS LINE ###### -LAST_MODIFIED_DATE=2013_01_11 -VERSION=0 +LAST_MODIFIED_DATE=2015_04_04 +VERSION=1 VERSION_STRING=${LAST_MODIFIED_DATE}_${VERSION} printUsage() { echo "###############################################################################" echo "#" - echo "# ghettoVCB-restore for ESX/ESXi 3.5, 4.x and 5.x" + echo "# ghettoVCB-restore for ESX/ESXi 3.5, 4.x, 5.x & 6.x" echo "# Author: William Lam" echo "# http://www.virtuallyghetto.com/" echo "# Documentation: http://communities.vmware.com/docs/DOC-8760" @@ -85,18 +85,19 @@ sanityCheck() { VMWARE_CMD=/bin/vim-cmd VMKFSTOOLS_CMD=/sbin/vmkfstools else - logger "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x, 5.x!" - echo "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x, 5.x!" + logger "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+, 5.x+ or 6.x!" + echo "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+, 5.x+ or 6.x!" exit fi ESX_VERSION=$(vmware -v | awk '{print $3}') case "${ESX_VERSION}" in + 6.0.0) VER=6; break;; 5.0.0|5.1.0|5.5.0) VER=5; break;; 4.0.0|4.1.0) VER=4; break;; 3.5.0|3i) VER=3; break;; - *) echo "You're not running ESX(i) 3.5, 4.x, 5.x!"; exit 1; break;; + *) echo "You're not running ESX(i) 3.5, 4.x, 5.x & 6.x!"; exit 1; break;; esac TAR="tar" @@ -159,11 +160,16 @@ ghettoVCBrestore() { fi #supports DIR or .TGZ from ghettoVCB.sh ONLY! + if [ ${VM_TO_RESTORE##*.} == 'gz' ]; then + logger "GZ found, extracting ..." + ${TAR} -xzf $VM_TO_RESTORE -C `dirname $VM_TO_RESTORE` + VM_TO_RESTORE=${VM_TO_RESTORE%.*} + fi if [ -d "${VM_TO_RESTORE}" ]; then #figure out the contents of the directory (*.vmdk,*-flat.vmdk,*.vmx) VM_ORIG_VMX=$(ls "${VM_TO_RESTORE}" | grep ".vmx") VM_VMDK_DESCRS=$(ls "${VM_TO_RESTORE}" | grep ".vmdk" | grep -v "\-flat.vmdk") - VMDKS_FOUND=$(grep -iE '(scsi|ide)' "${VM_TO_RESTORE}/${VM_ORIG_VMX}" | grep -i fileName | awk -F " " '{print $1}') + VMDKS_FOUND=$(grep -iE '(scsi|ide|sata)' "${VM_TO_RESTORE}/${VM_ORIG_VMX}" | grep -i fileName | awk -F " " '{print $1}') VM_FOLDER_NAME=$(echo "${VM_TO_RESTORE##*/}") # Default to original VM Display Name if custom name is not specified @@ -248,7 +254,7 @@ if [ ! "${IS_TGZ}" == "1" ]; then else #validates the datastore to restore is valid and available if [ ! -d "${DATASTORE_TO_RESTORE_TO}" ]; then - logger "ERROR: Unable to verify datastore locateion: \"${DATASTORE_TO_RESTORE_TO}\"! Ensure this exists" + logger "ERROR: Unable to verify datastore location: \"${DATASTORE_TO_RESTORE_TO}\"! Ensure this exists" #validates that all 4 required variables are defined before continuing elif [[ -z "${VM_RESTORE_VMX}" ]] && [[ -z "${VM_VMDK_DESCRS}" ]] && [[ -z "${VM_DISPLAY_NAME}" ]] && [[ -z "${VM_RESTORE_FOLDER_NAME}" ]]; then @@ -319,7 +325,7 @@ if [ ! "${IS_TGZ}" == "1" ]; then ADAPTER_FORMAT=$(grep -i "ddb.adapterType" "${SOURCE_VMDK}" | awk -F "=" '{print $2}' | sed -e 's/^[[:blank:]]*//;s/[[:blank:]]*$//;s/"//g') if [ ${RESTORE_DISK_FORMAT} -eq 1 ]; then - if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]]; then + if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] || [[ "${VER}" == "6" ]] ; then ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" -d zeroedthick "${DESTINATION_VMDK}" 2>&1 | tee "${REDIRECT}" else ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" "${DESTINATION_VMDK}" 2>&1 | tee "${REDIRECT}" @@ -332,7 +338,7 @@ if [ ! "${IS_TGZ}" == "1" ]; then ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" -d thin "${DESTINATION_VMDK}" 2>&1 | tee "${REDIRECT}" elif [ ${RESTORE_DISK_FORMAT} -eq 4 ]; then - if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]]; then + if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] || [[ "${VER}" == "6" ]] ; then ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" -d eagerzeroedthick "${DESTINATION_VMDK}" 2>&1 | tee "${REDIRECT}" else ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" "${DESTINATION_VMDK}" 2>&1 | tee "${REDIRECT}" diff --git a/ghettoVCB.conf b/ghettoVCB.conf index 21d7627..d0ea207 100644 --- a/ghettoVCB.conf +++ b/ghettoVCB.conf @@ -17,11 +17,13 @@ NFS_MOUNT=/nfsshare NFS_LOCAL_NAME=nfs_storage_backup NFS_VM_BACKUP_DIR=mybackups SNAPSHOT_TIMEOUT=15 +EMAIL_ALERT=0 EMAIL_LOG=0 EMAIL_SERVER=auroa.primp-industries.com EMAIL_SERVER_PORT=25 EMAIL_DELAY_INTERVAL=1 EMAIL_TO=auroa@primp-industries.com +EMAIL_ERRORS_TO= EMAIL_FROM=root@ghettoVCB WORKDIR_DEBUG=0 VM_SHUTDOWN_ORDER= diff --git a/ghettoVCB.sh b/ghettoVCB.sh index 6f93744..eb8a5d2 100755 --- a/ghettoVCB.sh +++ b/ghettoVCB.sh @@ -7,8 +7,8 @@ # User Definable Parameters ################################################################## -LAST_MODIFIED_DATE=2013_26_11 -VERSION=2 +LAST_MODIFIED_DATE=2015_11_06 #06.11.15 Original: LAST_MODIFIED_DATE=2015_05_06 +VERSION=2.7 #06.11.15 Original: VERSION=1 # directory that all VM backups should go (e.g. /vmfs/volumes/SAN_LUN1/mybackupdir) VM_BACKUP_VOLUME=/vmfs/volumes/mini-local-datastore-2/backups @@ -84,6 +84,8 @@ NFS_VM_BACKUP_DIR=mybackups # EMAIL CONFIGURATIONS # +# Email Alerting 1=yes, 0=no +EMAIL_ALERT=0 # Email log 1=yes, 0=no EMAIL_LOG=0 @@ -99,9 +101,12 @@ EMAIL_SERVER_PORT=25 # Email FROM EMAIL_FROM=root@ghettoVCB -# Email RCPT +# Comma seperated list of receiving email addresses EMAIL_TO=auroa@primp-industries.com +# Comma seperated list of additional receiving email addresses if status is not "OK" +EMAIL_ERRORS_TO= + # Comma separated list of VM startup/shutdown ordering VM_SHUTDOWN_ORDER= VM_STARTUP_ORDER= @@ -109,6 +114,12 @@ VM_STARTUP_ORDER= # RSYNC LINK 1=yes, 0 = no RSYNC_LINK=0 +# Extra options for ${VMKFSTOOLS_CMD} f.ex "-v 3" = verbose level 3 #01.10.14 +VMKFSTOOLS_CMD_OPTIONS= + +# Optional pause (seconds) between backup of vmdk's. 0 = no pause #16.10.14 +VMKFSTOOLS_PAUSE=0 + # DO NOT USE - UNTESTED CODE # Path to another location that should have backups rotated, # this is useful when your backups go to a temporary location @@ -139,7 +150,7 @@ VM_BACKUP_DIR_NAMING_CONVENTION="$(date +%F_%H-%M-%S)" printUsage() { echo "###############################################################################" echo "#" - echo "# ghettoVCB for ESX/ESXi 3.5, 4.x+ and 5.x" + echo "# ghettoVCB for ESX/ESXi 3.5, 4.x+, 5.x & 6.x" echo "# Author: William Lam" echo "# http://www.virtuallyghetto.com/" echo "# Documentation: http://communities.vmware.com/docs/DOC-8760" @@ -157,7 +168,7 @@ printUsage() { echo " -c VM configuration directory for VM backups" echo " -g Path to global ghettoVCB configuration file" echo " -l File to output logging" - echo " -w ghettoVCB work directory (default: /tmp/ghettoVCB.work)" + echo " -w ghettoVCB work directory (default: ${WORKDIR_DEFAULT})" echo " -d Debug level [info|debug|dryrun] (default: info)" echo echo "(e.g.)" @@ -208,10 +219,11 @@ sanityCheck() { exit 1 fi - # use of global ghettoVCB configuration - if [[ "${USE_GLOBAL_CONF}" -eq 1 ]] ; then - reConfigureGhettoVCBConfiguration "${GLOBAL_CONF}" - fi + #29.08.14 Commented out here because of "WORKDIR_DEBUG=1" and definition in calling place => moved into calling place. + ## use of global ghettoVCB configuration + ##if [[ "${USE_GLOBAL_CONF}" -eq 1 ]] ; then + ## reConfigureGhettoVCBConfiguration "${GLOBAL_CONF}" + ##fi # always log to STDOUT, use "> /dev/null" to ignore output LOG_TO_STDOUT=1 @@ -253,8 +265,8 @@ sanityCheck() { VMWARE_CMD=/bin/vim-cmd VMKFSTOOLS_CMD=/sbin/vmkfstools else - logger "info" "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+ or 5.0!" - echo "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+ or 5.0!" + logger "info" "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+, 5.x+ or 6.x!" + echo "ERROR: Unable to locate *vimsh*! You're not running ESX(i) 3.5+, 4.x+, 5.x+ or 6.x!" exit 1 fi @@ -262,10 +274,11 @@ sanityCheck() { ESX_RELEASE=$(uname -r) case "${ESX_VERSION}" in + 6.0.0) VER=6; break;; 5.0.0|5.1.0|5.5.0) VER=5; break;; 4.0.0|4.1.0) VER=4; break;; 3.5.0|3i) VER=3; break;; - *) echo "You're not running ESX(i) 3.5, 4.x, 5.x!"; exit 1; break;; + *) echo "You're not running ESX(i) 3.5, 4.x, 5.x & 6.x!"; exit 1; break;; esac NEW_VIMCMD_SNAPSHOT="no" @@ -278,17 +291,18 @@ sanityCheck() { elif [[ -f /bin/nc ]] ; then NC_BIN=/bin/nc fi - else + else #It is not possible to send email from this system EMAIL_LOG=0 + EMAIL_ALERT=0 #06.11.15 fi TAR="tar" [[ ! -f /bin/tar ]] && TAR="busybox tar" # Enable multiextent VMkernel module if disk format is 2gbsparse (disabled by default in 5.1) - if [[ "${DISK_BACKUP_FORMAT}" == "2gbsparse" ]] && [[ "${VER}" -eq 5 ]]; then + if [[ "${DISK_BACKUP_FORMAT}" == "2gbsparse" ]] && [[ "${VER}" -eq 5 ]] || [[ "${VER}" == "6" ]]; then esxcli system module list | grep multiextent > /dev/null 2>&1 - if [ $? -eq 1 ]; then + if [ $? -eq 1 ]; then logger "info" "multiextent VMkernel module is not loaded & is required for 2gbsparse, enabling ..." esxcli system module load -m multiextent fi @@ -332,6 +346,9 @@ captureDefaultConfigurations() { DEFAULT_VM_SHUTDOWN_ORDER="${VM_SHUTDOWN_ORDER}" DEFAULT_VM_STARTUP_ORDER="${VM_STARTUP_ORDER}" DEFAULT_RSYNC_LINK="${RSYNC_LINK}" + DEFAULT_BACKUP_FILES_CHMOD="${BACKUP_FILES_CHMOD}" + DEFAULT_VMKFSTOOLS_CMD_OPTIONS="${VMKFSTOOLS_CMD_OPTIONS}" #01.10.14 + DEFAULT_VMKFSTOOLS_PAUSE="${VMKFSTOOLS_PAUSE}" #16.10.14 } useDefaultConfigurations() { @@ -352,7 +369,10 @@ useDefaultConfigurations() { WORKDIR_DEBUG="${DEFAULT_WORKDIR_DEBUG}" VM_SHUTDOWN_ORDER="${DEFAULT_VM_SHUTDOWN_ORDER}" VM_STARTUP_ORDER="${DEFAULT_VM_STARTUP_ORDER}" - RSYNC_LINK="${RSYNC_LINK}" + RSYNC_LINK="${DEFAULT_RSYNC_LINK}" #16.10.14 Added prefix "DEFAULT_" + BACKUP_FILES_CHMOD="${BACKUP_FILES_CHMOD}" + VMKFSTOOLS_CMD_OPTIONS="${DEFAULT_VMKFSTOOLS_CMD_OPTIONS}" #01.10.14 + VMKFSTOOLS_PAUSE="${DEFAULT_VMKFSTOOLS_PAUSE}" #16.10.14 } reConfigureGhettoVCBConfiguration() { @@ -405,7 +425,10 @@ findVMDK() { getVMDKs() { #get all VMDKs listed in .vmx file - VMDKS_FOUND=$(grep -iE '(^scsi|^ide)' "${VMX_PATH}" | grep -i fileName | awk -F " " '{print $1}') + VMDKS_FOUND=$(grep -iE '(^scsi|^ide|^sata)' "${VMX_PATH}" | grep -i fileName | awk -F " " '{print $1}') + + VMDKS= + INDEP_VMDKS= TMP_IFS=${IFS} IFS=${ORIG_IFS} @@ -503,7 +526,11 @@ dumpVMConfigurations() { logger "info" "CONFIG - VM_SHUTDOWN_ORDER = ${VM_SHUTDOWN_ORDER}" logger "info" "CONFIG - VM_STARTUP_ORDER = ${VM_STARTUP_ORDER}" logger "info" "CONFIG - RSYNC_LINK = ${RSYNC_LINK}" + logger "info" "CONFIG - BACKUP_FILES_CHMOD = ${BACKUP_FILES_CHMOD}" logger "info" "CONFIG - EMAIL_LOG = ${EMAIL_LOG}" + logger "info" "CONFIG - VMKFSTOOLS_CMD_OPTIONS = ${VMKFSTOOLS_CMD_OPTIONS}" #01.10.14 + logger "info" "CONFIG - VMKFSTOOLS_PAUSE = ${VMKFSTOOLS_PAUSE}" #16.10.14 + if [[ "${EMAIL_LOG}" -eq 1 ]]; then logger "info" "CONFIG - EMAIL_SERVER = ${EMAIL_SERVER}" logger "info" "CONFIG - EMAIL_SERVER_PORT = ${EMAIL_SERVER_PORT}" @@ -511,6 +538,11 @@ dumpVMConfigurations() { logger "info" "CONFIG - EMAIL_FROM = ${EMAIL_FROM}" logger "info" "CONFIG - EMAIL_TO = ${EMAIL_TO}" logger "info" "CONFIG - WORKDIR_DEBUG = ${WORKDIR_DEBUG}" + + #29.08.14 Info about workdir + if [[ "${WORKDIR_DEBUG}" -eq 1 ]]; then + logger "info" "CONFIG - WORKDIR = ${WORKDIR}" + fi fi logger "info" "" } @@ -578,8 +610,16 @@ checkVMBackupRotation() { VM_BACKUP_ROTATION_COUNT=1 fi - LIST_BACKUPS=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}") - BACKUPS_TO_KEEP=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" | head -"${VM_BACKUP_ROTATION_COUNT}") + if [[ ${VM_VMDK_FAILED} -ne 0 ]] ; then #16.10.14-> + # Error with backup of VMDKs of this VM, remove latest falty backup if any. + LIST_BACKUPS=$(ls -tr "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}") + BACKUPS_TO_KEEP=$(ls -tr "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" | head -"${VM_BACKUP_ROTATION_COUNT}") + logger "info" "This is falty backup of ${VM_NAME}: ${VM_NAME}-${VM_BACKUP_DIR_NAMING_CONVENTION}. These backups will not be removed: ${BACKUPS_TO_KEEP}" + else #16.10.14<- + # Backup of VMDK's of this VM was succesfull, remove oldest backup if any. + LIST_BACKUPS=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}") + BACKUPS_TO_KEEP=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" | head -"${VM_BACKUP_ROTATION_COUNT}") + fi ORIG_IFS=${IFS} IFS=' @@ -625,7 +665,12 @@ storageInfo() { SECTION=$1 #SOURCE DATASTORE - SRC_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${VMFS_VOLUME}" | grep -i "capacity" | awk '{print $3}' | sed 's/,//g') + + #22.10.14 Removed parameter '-i' because with it, grep finds two values in VMware 5.5.0: "maxVirtualDiskCapacity" and "capacity" + # resulting wrong value as value of 'SRC_DATASTORE_FREE:'. + #SRC_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${VMFS_VOLUME}" | grep -i "capacity" | awk '{print $3}' | sed 's/,//g') + SRC_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${VMFS_VOLUME}" | grep "capacity" | awk '{print $3}' | sed 's/,//g') + SRC_DATASTORE_FREE=$($VMWARE_CMD hostsvc/datastore/info "${VMFS_VOLUME}" | grep -i "freespace" | awk '{print $3}' | sed 's/,//g') SRC_DATASTORE_BLOCKSIZE=$($VMWARE_CMD hostsvc/datastore/info "${VMFS_VOLUME}" | grep -i blockSizeMb | awk '{print $3}' | sed 's/,//g') if [[ -z ${SRC_DATASTORE_BLOCKSIZE} ]] ; then @@ -645,7 +690,11 @@ storageInfo() { #DESTINATION DATASTORE DST_VOL_1=$(echo "${VM_BACKUP_VOLUME#/*/*/}") DST_DATASTORE=$(echo "${DST_VOL_1%%/*}") - DST_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${DST_DATASTORE}" | grep -i "capacity" | awk '{print $3}' | sed 's/,//g') + + #22.10.14 Removed parameter '-i' because with it, grep finds two values in VMware 5.5.0: "maxVirtualDiskCapacity" and "capacity" + #DST_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${DST_DATASTORE}" | grep -i "capacity" | awk '{print $3}' | sed 's/,//g') + DST_DATASTORE_CAPACITY=$($VMWARE_CMD hostsvc/datastore/info "${DST_DATASTORE}" | grep "capacity" | awk '{print $3}' | sed 's/,//g') + DST_DATASTORE_FREE=$($VMWARE_CMD hostsvc/datastore/info "${DST_DATASTORE}" | grep -i "freespace" | awk '{print $3}' | sed 's/,//g') DST_DATASTORE_BLOCKSIZE=$($VMWARE_CMD hostsvc/datastore/info "${DST_DATASTORE}" | grep -i blockSizeMb | awk '{print $3}' | sed 's/,//g') @@ -756,6 +805,7 @@ ghettoVCB() { VM_OK=0 VM_FAILED=0 VMDK_FAILED=0 + PROBLEM_VMS= dumpHostInfo @@ -765,7 +815,7 @@ ghettoVCB() { #1 = readonly #0 = readwrite logger "debug" "Mounting NFS: ${NFS_SERVER}:${NFS_MOUNT} to /vmfs/volume/${NFS_LOCAL_NAME}" - if [ ${ESX_RELEASE} == "5.5.0" ]; then + if [[ ${ESX_RELEASE} == "5.5.0" ]] || [[ ${ESX_RELEASE} == "6.0.0" ]] ; then ${VMWARE_CMD} hostsvc/datastore/nas_create "${NFS_LOCAL_NAME}" "${NFS_VERSION}" "${NFS_MOUNT}" 0 "${NFS_SERVER}" else ${VMWARE_CMD} hostsvc/datastore/nas_create "${NFS_LOCAL_NAME}" "${NFS_SERVER}" "${NFS_MOUNT}" 0 @@ -802,7 +852,7 @@ ghettoVCB() { powerOff "${VM_NAME}" "${VM_ID}" if [[ ${POWER_OFF_EC} -eq 1 ]]; then logger "debug" "Error unable to shutdown VM ${VM_NAME}\n" - exit 1 + PROBLEM_VMS="${PROBLEM_VMS} ${VM_NAME}" fi done @@ -815,6 +865,16 @@ ghettoVCB() { grep -E "^${VM_NAME}" "${VM_EXCLUSION_FILE}" > /dev/null 2>&1 if [[ $? -eq 0 ]] ; then IGNORE_VM=1 + #VM_FAILED=0 #Excluded VM is NOT a failure. No need to set here, but listed for clarity + fi + fi + + if [[ "${IGNORE_VM}" -eq 0 ]] && [[ -n "${PROBLEM_VMS}" ]] ; then + if [[ "${PROBLEM_VMS/$VM_NAME}" != "$PROBLEM_VMS" ]] ; then + logger "info" "Ignoring ${VM_NAME} as a problem VM\n" + IGNORE_VM=1 + #A VM ignored due to a problem, should be treated as a failure + VM_FAILED=1 fi fi @@ -840,9 +900,9 @@ ghettoVCB() { storageInfo "before" fi - #ignore VM as it's in the exclusion list + #ignore VM as it's in the exclusion list or was on problem list if [[ "${IGNORE_VM}" -eq 1 ]] ; then - logger "debug" "Ignoring ${VM_NAME} for backup since its located in exclusion list\n" + logger "debug" "Ignoring ${VM_NAME} for backup since it is located in exclusion file or problem list\n" #checks to see if we can pull out the VM_ID elif [[ -z ${VM_ID} ]] ; then logger "info" "ERROR: failed to locate and extract VM_ID for ${VM_NAME}!\n" @@ -870,13 +930,15 @@ ghettoVCB() { done HAS_INDEPENDENT_DISKS=0 - logger "dryrun" "INDEPENDENT VMDK(s): " - for k in ${INDEP_VMDKS}; do - HAS_INDEPENDENT_DISKS=1 - K_VMDK=$(echo "${k}" | awk -F "###" '{print $1}') - K_VMDK_SIZE=$(echo "${k}" | awk -F "###" '{print $2}') - logger "dryrun" "\t${K_VMDK}\t${K_VMDK_SIZE} GB" - done + if [[ ! -z ${INDEP_VMDKS} ]] ; then #27.08.14 Added "if..; then" and "fi" + logger "dryrun" "INDEPENDENT VMDK(s):" + for k in ${INDEP_VMDKS}; do + HAS_INDEPENDENT_DISKS=1 + K_VMDK=$(echo "${k}" | awk -F "###" '{print $1}') + K_VMDK_SIZE=$(echo "${k}" | awk -F "###" '{print $2}') + logger "dryrun" "\t${K_VMDK}\t${K_VMDK_SIZE} GB" + done + fi IFS="${OLD_IFS}" VMDKS="" @@ -884,7 +946,7 @@ ghettoVCB() { logger "dryrun" "TOTAL_VM_SIZE_TO_BACKUP: ${TOTAL_VM_SIZE} GB" if [[ ${HAS_INDEPENDENT_DISKS} -eq 1 ]] ; then - logger "dryrun" "Snapshots can not be taken for indepdenent disks!" + logger "dryrun" "Snapshots can not be taken for independent disks!" #05.09.14 logger "dryrun" "THIS VIRTUAL MACHINE WILL NOT HAVE ALL ITS VMDKS BACKED UP!" fi @@ -983,7 +1045,7 @@ ghettoVCB() { startTimer SNAP_SUCCESS=1 - VM_VMDK_FAILED=0 + VM_VMDK_FAILED=0 #Default status for backup of VMDK's of this VM is "succesfull" #powered on VMs only if [[ ! ${POWER_VM_DOWN_BEFORE_BACKUP} -eq 1 ]] && [[ "${ORGINAL_VM_POWER_STATE}" != "Powered off" ]]; then @@ -998,7 +1060,7 @@ ghettoVCB() { if [[ ${START_ITERATION} -ge ${SNAPSHOT_TIMEOUT} ]] ; then logger "info" "Snapshot timed out, failed to create snapshot: \"${SNAPSHOT_NAME}\" for ${VM_NAME}" SNAP_SUCCESS=0 - echo "ERROR: Unable to backup ${VM_NAME} due to snapshot creation" >> ${VM_BACKUP_DIR}/STATUS.error + [[ -d ${VM_BACKUP_DIR} ]] && echo "ERROR: Unable to backup ${VM_NAME} due to snapshot creation" >> ${VM_BACKUP_DIR}/STATUS.error break fi @@ -1018,7 +1080,8 @@ ghettoVCB() { findVMDK "${VMDK}" - if [[ $isVMDKFound -eq 1 ]] || [[ "${VMDK_FILES_TO_BACKUP}" == "all" ]]; then + #23.10.14 Continue backup of vm only if backup of previous vmdk's were succesfull (VM_VMDK_FAILED=0) + if [[ ${VM_VMDK_FAILED} -eq 0 ]] && ( [[ $isVMDKFound -eq 1 ]] || [[ "${VMDK_FILES_TO_BACKUP}" == "all" ]] ) ; then #added this section to handle VMDK(s) stored in different datastore than the VM echo ${VMDK} | grep "^/vmfs/volumes" > /dev/null 2>&1 if [[ $? -eq 0 ]] ; then @@ -1038,7 +1101,7 @@ ghettoVCB() { if [[ $? -eq 1 ]] ; then FORMAT_OPTION="UNKNOWN" if [[ "${DISK_BACKUP_FORMAT}" == "zeroedthick" ]] ; then - if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] ; then + if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] || [[ "${VER}" == "6" ]] ; then FORMAT_OPTION="zeroedthick" else FORMAT_OPTION="" @@ -1048,7 +1111,7 @@ ghettoVCB() { elif [[ "${DISK_BACKUP_FORMAT}" == "thin" ]] ; then FORMAT_OPTION="thin" elif [[ "${DISK_BACKUP_FORMAT}" == "eagerzeroedthick" ]] ; then - if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] ; then + if [[ "${VER}" == "4" ]] || [[ "${VER}" == "5" ]] || [[ "${VER}" == "6" ]] ; then FORMAT_OPTION="eagerzeroedthick" else FORMAT_OPTION="" @@ -1059,6 +1122,22 @@ ghettoVCB() { logger "info" "ERROR: wrong DISK_BACKUP_FORMAT \"${DISK_BACKUP_FORMAT}\ specified for ${VM_NAME}" VM_VMDK_FAILED=1 else + + #16.10.14 To debug "file already exist" error + if [[ "${LOG_LEVEL}" == "debug" ]] ; then + ( + echo "======= -- debug:" + echo VMDK:${VMDK} VMX_DIR:${VMX_DIR} VM_BACKUP_DIR:${VM_BACKUP_DIR} VMX_PATH:${VMX_PATH} VMDKS:${VMDKS} + echo "======= -- debug:" + echo Contents of directory ${VMX_DIR} for ${VMDK} before issuing VMKFSTOOLS_CMD ... + ls -laL ${VMX_DIR} + echo "======= -- debug:" + echo Contents of backup directory ${VM_BACKUP_DIR} for ${VMDK} before issuing VMKFSTOOLS_CMD ... + ls -laL ${VM_BACKUP_DIR} + echo "=======" + ) >> "${REDIRECT}" 2>&1 + fi + VMDK_OUTPUT=$(mktemp ${WORKDIR}/ghettovcb.XXXXXX) tail -f "${VMDK_OUTPUT}" & TAIL_PID=$! @@ -1066,11 +1145,12 @@ ghettoVCB() { ADAPTER_FORMAT=$(grep -i "ddb.adapterType" "${SOURCE_VMDK}" | awk -F "=" '{print $2}' | sed -e 's/^[[:blank:]]*//;s/[[:blank:]]*$//;s/"//g') if [[ -z "${FORMAT_OPTION}" ]] ; then - logger "debug" "${VMKFSTOOLS_CMD} -i \"${SOURCE_VMDK}\" -a \"${ADAPTER_FORMAT}\" \"${DESTINATION_VMDK}\"" - ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" "${DESTINATION_VMDK}" > "${VMDK_OUTPUT}" 2>&1 + #01.10.14 Added ${VMKFSTOOLS_CMD_OPTIONS}" below + logger "debug" "${VMKFSTOOLS_CMD} -i \"${SOURCE_VMDK}\" -a \"${ADAPTER_FORMAT}\" \"${DESTINATION_VMDK}\" ${VMKFSTOOLS_CMD_OPTIONS}" + ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" "${DESTINATION_VMDK}" ${VMKFSTOOLS_CMD_OPTIONS} > "${VMDK_OUTPUT}" 2>&1 else - logger "debug" "${VMKFSTOOLS_CMD} -i \"${SOURCE_VMDK}\" -a \"${ADAPTER_FORMAT}\" -d \"${FORMAT_OPTION}\" \"${DESTINATION_VMDK}\"" - ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" -d "${FORMAT_OPTION}" "${DESTINATION_VMDK}" > "${VMDK_OUTPUT}" 2>&1 + logger "debug" "${VMKFSTOOLS_CMD} -i \"${SOURCE_VMDK}\" -a \"${ADAPTER_FORMAT}\" -d \"${FORMAT_OPTION}\" \"${DESTINATION_VMDK}\" ${VMKFSTOOLS_CMD_OPTIONS}" + ${VMKFSTOOLS_CMD} -i "${SOURCE_VMDK}" -a "${ADAPTER_FORMAT}" -d "${FORMAT_OPTION}" "${DESTINATION_VMDK}" ${VMKFSTOOLS_CMD_OPTIONS} > "${VMDK_OUTPUT}" 2>&1 fi VMDK_EXIT_CODE=$? @@ -1078,20 +1158,45 @@ ghettoVCB() { cat "${VMDK_OUTPUT}" >> "${REDIRECT}" echo >> "${REDIRECT}" echo + + #16.10.14 Keep output messages of VMKFSTOOLS_CMD + if [[ "${LOG_LEVEL}" == "debug" ]] ; then + cp -p "${VMDK_OUTPUT}" "${WORKDIR}/ghettovcb_${VMDK}.log" + fi + rm "${VMDK_OUTPUT}" if [[ "${VMDK_EXIT_CODE}" != 0 ]] ; then logger "info" "ERROR: error in backing up of \"${SOURCE_VMDK}\" for ${VM_NAME}" VM_VMDK_FAILED=1 fi + + if [[ "${LOG_LEVEL}" == "debug" ]] ; then + ( + echo "======= -- debug:" + echo Contents of backup directory ${VM_BACKUP_DIR} for ${VMDK} after VMKFSTOOLS_CMD \(VM_VMDK_FAILED=${VM_VMDK_FAILED}\) ... + ls -laL ${VM_BACKUP_DIR} + echo "=======" + ) >> "${REDIRECT}" 2>&1 + fi + + #16.10.14 Optional pause between backup of different vmdk's (not with the last one or single VMDK) + if [[ ${VMKFSTOOLS_PAUSE} -gt 0 ]] && [[ "$(echo ${VMDKS} | awk -F " " '{print $(NF)}' | awk -F "###" '{print $1}')" != "${VMDK}" ]]; then + logger "info" "Pause ${VMKFSTOOLS_PAUSE} s. after backup of \"${VMDK}\" for ${VM_NAME} ..." + sleep ${VMKFSTOOLS_PAUSE} + fi fi else logger "info" "WARNING: A physical RDM \"${SOURCE_VMDK}\" was found for ${VM_NAME}, which will not be backed up" VM_VMDK_FAILED=1 fi - fi + elif [[ ${VM_VMDK_FAILED} -ne 0 ]] ; then #23.10.14-> + logger "info" "WARNING: Backup of \"${VMDK}\" for \"${VM_NAME}\" terminated because backup of some other vmdk failed earlier" + fi #23.10.14<- done IFS="${OLD_IFS}" + else + VM_VMDK_FAILED=1 #16.10.14 Snapshot failed => backup of VMDKs failed too fi #powered on VMs only w/snapshots @@ -1116,9 +1221,112 @@ ghettoVCB() { ${VMWARE_CMD} vmsvc/power.on ${VM_ID} > /dev/null 2>&1 fi + #16.10.14-> Commented out, moved to end of this block and modified + #TMP_IFS=${IFS} + #IFS=${ORIG_IFS} + # + #if [[ ${ENABLE_COMPRESSION} -eq 1 ]] ; then + # COMPRESSED_ARCHIVE_FILE="${BACKUP_DIR}/${VM_NAME}-${VM_BACKUP_DIR_NAMING_CONVENTION}.gz" + # + # logger "info" "Compressing VM backup \"${COMPRESSED_ARCHIVE_FILE}\"..." + # ${TAR} -cz -C "${BACKUP_DIR}" "${VM_NAME}-${VM_BACKUP_DIR_NAMING_CONVENTION}" -f "${COMPRESSED_ARCHIVE_FILE}" + # + # # verify compression + # if [[ $? -eq 0 ]] && [[ -f "${COMPRESSED_ARCHIVE_FILE}" ]]; then + # logger "info" "Successfully compressed backup for ${VM_NAME}!\n" + # COMPRESSED_OK=1 + # else + # logger "info" "Error in compressing ${VM_NAME}!\n" + # COMPRESSED_OK=0 + # fi + # rm -rf "${VM_BACKUP_DIR}" + # checkVMBackupRotation "${BACKUP_DIR}" "${VM_NAME}" + #else + # checkVMBackupRotation "${BACKUP_DIR}" "${VM_NAME}" + #fi + # + #IFS=${TMP_IFS} + #VMDKS="" #27.08.14 Commented out, moved to end of this block + #INDEP_VMDKS="" #27.08.14 Commented out, moved to end of this block + #16.10.14<- + + #endTimer #16.10.14 Commented out, moved to end of this block + + if [[ ${SNAP_SUCCESS} -ne 1 ]] ; then + logger "info" "ERROR: Unable to backup ${VM_NAME} due to snapshot creation!\n" + #16.10.14 #[[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || \ + echo "ERROR: Unable to backup ${VM_NAME} due to snapshot creation" >> ${VM_BACKUP_DIR}/STATUS.error + VM_FAILED=1 + elif [[ ${VM_VMDK_FAILED} -ne 0 ]] ; then + logger "info" "ERROR: Unable to backup ${VM_NAME} due to error in VMDK backup!\n" + #16.10.14 #[[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || \ + echo "ERROR: Unable to backup ${VM_NAME} due to error in VMDK backup" >> ${VM_BACKUP_DIR}/STATUS.error + VMDK_FAILED=1 + elif [[ ${VM_HAS_INDEPENDENT_DISKS} -eq 1 ]] ; then + logger "info" "WARN: ${VM_NAME} has some Independent VMDKs that can not be backed up!\n"; + #16.10.14 #[[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || \ + echo "WARN: ${VM_NAME} has some Independent VMDKs that can not be backed up" > ${VM_BACKUP_DIR}/STATUS.warn + + #27.08.14 To get proper "Final status" message, if there has been at least one normal VMDK in addition to Independant VMDKs, + #backup of virtual machine has been partially succesfull: + if [[ ! -z ${VMDKS} ]] ; then + VM_OK=1 + fi + + #Some of the VMDKs failed because of Independant definition: + VMDK_FAILED=1 + + #16.10.14-> Commented out, moved to end of this block + #experimental, create symlink for the very last backup to support rsync functionality for additinal replication + #if [[ "${RSYNC_LINK}" -eq 1 ]] ; then + # SYMLINK_DST=${VM_BACKUP_DIR} + # if [[ ${ENABLE_COMPRESSION} -eq 1 ]]; then + # SYMLINK_DST1="${RSYNC_LINK_DIR}.gz" + # else + # SYMLINK_DST1=${RSYNC_LINK_DIR} + # fi + # SYMLINK_SRC="$(echo "${SYMLINK_DST%*-*-*-*_*-*-*}")-symlink" + # logger "info" "Creating symlink \"${SYMLINK_SRC}\" to \"${SYMLINK_DST1}\"" + # ln -sf "${SYMLINK_DST1}" "${SYMLINK_SRC}" + #fi + + #storage info after backup + #storageInfo "after" #16.10.14<- Commented out, moved to end of this block + else + logger "info" "Successfully completed backup for ${VM_NAME}!\n" + #16.10.14 #[[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || \ + echo "Successfully completed backup" > ${VM_BACKUP_DIR}/STATUS.ok + VM_OK=1 + + #16.10.14-> Commented out, moved to end of this block + #experimental, create symlink for the very last backup to support rsync functionality for additinal replication + #if [[ "${RSYNC_LINK}" -eq 1 ]] ; then + # SYMLINK_DST=${VM_BACKUP_DIR} + # if [[ ${ENABLE_COMPRESSION} -eq 1 ]] ; then + # SYMLINK_DST1="${RSYNC_LINK_DIR}.gz" + # else + # SYMLINK_DST1=${RSYNC_LINK_DIR} + # fi + # SYMLINK_SRC="$(echo "${SYMLINK_DST%*-*-*-*_*-*-*}")-symlink" + # logger "info" "Creating symlink \"${SYMLINK_SRC}\" to \"${SYMLINK_DST1}\"" + # ln -sf "${SYMLINK_DST1}" "${SYMLINK_SRC}" + #fi + + #if [[ "${BACKUP_FILES_CHMOD}" != "" ]] + #then + # chmod -R "${BACKUP_FILES_CHMOD}" "${VM_BACKUP_DIR}" + #fi + + #storage info after backup + #storageInfo "after" #16.10.14<- Commented out, moved to end of this block + fi + + #16.01.14-> Moved here from abowe TMP_IFS=${IFS} IFS=${ORIG_IFS} - if [[ ${ENABLE_COMPRESSION} -eq 1 ]] ; then + + #16.01.14 Added test for "VM_VMDK_FAILED" => do not compress faulty backup, it will be removed by checkVMBackupRotation. + if [[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ ${VM_VMDK_FAILED} -eq 0 ]] ; then COMPRESSED_ARCHIVE_FILE="${BACKUP_DIR}/${VM_NAME}-${VM_BACKUP_DIR_NAMING_CONVENTION}.gz" logger "info" "Compressing VM backup \"${COMPRESSED_ARCHIVE_FILE}\"..." @@ -1137,61 +1345,36 @@ ghettoVCB() { else checkVMBackupRotation "${BACKUP_DIR}" "${VM_NAME}" fi - IFS=${TMP_IFS} - VMDKS="" - INDEP_VMDKS="" - endTimer - if [[ ${SNAP_SUCCESS} -ne 1 ]] ; then - logger "info" "ERROR: Unable to backup ${VM_NAME} due to snapshot creation!\n" - [[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || echo "ERROR: Unable to backup ${VM_NAME} due to snapshot creation" >> ${VM_BACKUP_DIR}/STATUS.error - VM_FAILED=1 - elif [[ ${VM_VMDK_FAILED} -ne 0 ]] ; then - logger "info" "ERROR: Unable to backup ${VM_NAME} due to error in VMDK backup!\n" - [[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || echo "ERROR: Unable to backup ${VM_NAME} due to error in VMDK backup" >> ${VM_BACKUP_DIR}/STATUS.error - VMDK_FAILED=1 - elif [[ ${VM_HAS_INDEPENDENT_DISKS} -eq 1 ]] ; then - logger "info" "WARN: ${VM_NAME} has some Independent VMDKs that can not be backed up!\n"; - [[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || echo "WARN: ${VM_NAME} has some Independent VMDKs that can not be backed up" > ${VM_BACKUP_DIR}/STATUS.warn - VMDK_FAILED=1 - #experimental - #create symlink for the very last backup to support rsync functionality for additinal replication - if [[ "${RSYNC_LINK}" -eq 1 ]] ; then - SYMLINK_DST=${VM_BACKUP_DIR} - if [[ ${ENABLE_COMPRESSION} -eq 1 ]]; then - SYMLINK_DST1="${RSYNC_LINK_DIR}.gz" - else - SYMLINK_DST1=${RSYNC_LINK_DIR} - fi - SYMLINK_SRC="$(echo "${SYMLINK_DST%*-*-*-*_*-*-*}")-symlink" - logger "info" "Creating symlink \"${SYMLINK_SRC}\" to \"${SYMLINK_DST1}\"" - ln -sf "${SYMLINK_DST1}" "${SYMLINK_SRC}" + IFS=${TMP_IFS} + #16.10.14<- Moved here from abowe + + #16.01.14-> Moved here from abowe + #experimental, create symlink for the very last backup to support rsync functionality for additional replication + if [[ "${RSYNC_LINK}" -eq 1 ]] && [[ ${VM_VMDK_FAILED} -eq 0 ]] ; then + SYMLINK_DST=${VM_BACKUP_DIR} + if [[ ${ENABLE_COMPRESSION} -eq 1 ]] ; then + SYMLINK_DST1="${RSYNC_LINK_DIR}.gz" + else + SYMLINK_DST1=${RSYNC_LINK_DIR} fi + SYMLINK_SRC="$(echo "${SYMLINK_DST%*-*-*-*_*-*-*}")-symlink" + logger "info" "Creating symlink \"${SYMLINK_SRC}\" to \"${SYMLINK_DST1}\"" + ln -sf "${SYMLINK_DST1}" "${SYMLINK_SRC}" + fi - #storage info after backup - storageInfo "after" - else - logger "info" "Successfully completed backup for ${VM_NAME}!\n" - [[ ${ENABLE_COMPRESSION} -eq 1 ]] && [[ $COMPRESSED_OK -eq 1 ]] || echo "Successfully completed backup" > ${VM_BACKUP_DIR}/STATUS.ok - VM_OK=1 + if [[ "${BACKUP_FILES_CHMOD}" != "" ]] + then + chmod -R "${BACKUP_FILES_CHMOD}" "${VM_BACKUP_DIR}" + fi + + storageInfo "after" + endTimer + #16.01.14<- Moved here from abowe - #experimental - #create symlink for the very last backup to support rsync functionality for additinal replication - if [[ "${RSYNC_LINK}" -eq 1 ]] ; then - SYMLINK_DST=${VM_BACKUP_DIR} - if [[ ${ENABLE_COMPRESSION} -eq 1 ]] ; then - SYMLINK_DST1="${RSYNC_LINK_DIR}.gz" - else - SYMLINK_DST1=${RSYNC_LINK_DIR} - fi - SYMLINK_SRC="$(echo "${SYMLINK_DST%*-*-*-*_*-*-*}")-symlink" - logger "info" "Creating symlink \"${SYMLINK_SRC}\" to \"${SYMLINK_DST1}\"" - ln -sf "${SYMLINK_DST1}" "${SYMLINK_SRC}" - fi + VMDKS="" #27.08.14 Moved here from abowe + INDEP_VMDKS="" #27.08.14 Moved here from abowe - #storage info after backup - storageInfo "after" - fi else if [[ ${CONTINUE_TO_BACKUP} -eq 0 ]] ; then logger "info" "ERROR: Failed to backup ${VM_NAME}!\n" @@ -1218,7 +1401,7 @@ ghettoVCB() { #fi unset IFS - if [[ ${#VM_STARTUP_ORDER} -gt 0 ]]; then + if [[ ${#VM_STARTUP_ORDER} -gt 0 ]] && [[ "${LOG_LEVEL}" != "dryrun" ]]; then logger "debug" "VM Startup Order: ${VM_STARTUP_ORDER}\n" IFS="," for VM_NAME in ${VM_STARTUP_ORDER}; do @@ -1295,13 +1478,38 @@ buildHeaders() { sendMail() { #close email message - if [[ "${EMAIL_LOG}" -eq 1 ]] ; then + if [[ "${EMAIL_LOG}" -eq 1 ]] || [[ "${EMAIL_ALERT}" -eq 1]] ; then + SMTP=1 #validate firewall has email port open for ESXi 5 - if [[ "${VER}" == "5" ]] ; then + if [[ "${VER}" == "5" ]] || [[ "${VER}" == "6" ]] ; then /sbin/esxcli network firewall ruleset rule list | grep "${EMAIL_SERVER_PORT}" > /dev/null 2>&1 if [[ $? -eq 1 ]] ; then logger "info" "ERROR: Please enable firewall rule for email traffic on port ${EMAIL_SERVER_PORT}\n" logger "info" "Please refer to ghettoVCB documentation for ESXi 5 firewall configuration\n" + SMTP=0 + fi + fi + fi + if [[ "${SMTP}" -eq 1 ]] ; then + + if [ "${EXIT}" -ne 0 ] && [ "${LOG_STATUS}" = "OK" ] ; then + LOG_STATUS="ERROR" + # for i in ${EMAIL_TO}; do + # buildHeaders ${i} + # cat "${EMAIL_LOG_CONTENT}" |while read L; do sleep "${EMAIL_DELAY_INTERVAL}"; echo $L; done | "${NC_BIN}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" > /dev/null 2>&1 + # #"${NC_BIN}" -i "${EMAIL_DELAY_INTERVAL}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" < "${EMAIL_LOG_CONTENT}" > /dev/null 2>&1 + # if [[ $? -eq 1 ]] ; then + # logger "info" "ERROR: Failed to email log output to ${EMAIL_SERVER}:${EMAIL_SERVER_PORT} to ${EMAIL_TO}\n" + # fi + # done + fi + + + if [ "${EMAIL_ERRORS_TO}" != "" ] && [ "${LOG_STATUS}" != "OK" ] ; then + if [ "${EMAIL_TO}" == "" ] ; then + EMAIL_TO="${EMAIL_ERRORS_TO}" + else + EMAIL_TO="${EMAIL_TO},${EMAIL_ERRORS_TO}" fi fi @@ -1311,6 +1519,7 @@ sendMail() { IFS=',' for i in ${EMAIL_TO}; do buildHeaders ${i} + # cat "${EMAIL_LOG_CONTENT}" |while read L; do sleep "${EMAIL_DELAY_INTERVAL}"; echo $L; done | "${NC_BIN}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" > /dev/null 2>&1 "${NC_BIN}" -i "${EMAIL_DELAY_INTERVAL}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" < "${EMAIL_LOG_CONTENT}" > /dev/null 2>&1 if [[ $? -eq 1 ]] ; then logger "info" "ERROR: Failed to email log output to ${EMAIL_SERVER}:${EMAIL_SERVER_PORT} to ${EMAIL_TO}\n" @@ -1319,6 +1528,7 @@ sendMail() { unset IFS else buildHeaders ${EMAIL_TO} + # cat "${EMAIL_LOG_CONTENT}" |while read L; do sleep "${EMAIL_DELAY_INTERVAL}"; echo $L; done | "${NC_BIN}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" > /dev/null 2>&1 "${NC_BIN}" -i "${EMAIL_DELAY_INTERVAL}" "${EMAIL_SERVER}" "${EMAIL_SERVER_PORT}" < "${EMAIL_LOG_CONTENT}" > /dev/null 2>&1 if [[ $? -eq 1 ]] ; then logger "info" "ERROR: Failed to email log output to ${EMAIL_SERVER}:${EMAIL_SERVER_PORT} to ${EMAIL_TO}\n" @@ -1337,6 +1547,7 @@ USE_VM_CONF=0 USE_GLOBAL_CONF=0 BACKUP_ALL_VMS=0 EXCLUDE_SOME_VMS=0 +WORKDIR_DEFAULT=/tmp/ghettoVCB.work #29.08.14 # quick sanity check on the number of arguments if [[ $# -lt 1 ]] || [[ $# -gt 12 ]]; then @@ -1353,13 +1564,13 @@ while getopts ":af:c:g:w:m:l:d:e:" ARGS; do ;; a) BACKUP_ALL_VMS=1 - VM_FILE='${WORKDIR}/vm-input-list' + VM_FILE='vm-input-list' #29.08.14 Was ${WORKDIR}/vm-input-list, related to cmdline location of "-w" ;; f) VM_FILE="${OPTARG}" ;; m) - VM_FILE='${WORKDIR}/vm-input-list' + VM_FILE='vm-input-list' #29.08.14 Was ${WORKDIR}/vm-input-list, related to cmdline location of "-w" VM_ARG="${OPTARG}" ;; e) @@ -1386,11 +1597,17 @@ while getopts ":af:c:g:w:m:l:d:e:" ARGS; do ;; *) printUsage + exit 1 #23.08.14 Do not continue in case of unkown parameter ;; esac done -WORKDIR=${WORKDIR:-"/tmp/ghettoVCB.work"} +WORKDIR=${WORKDIR:-"${WORKDIR_DEFAULT}"} + +#29.08.14 Now "-w" parameter does not need to be located before parameters -a and -m on command line. +if [[ "${VM_FILE}" == "vm-input-list" ]]; then + VM_FILE=${WORKDIR}/${VM_FILE} +fi EMAIL_LOG_HEADER=${WORKDIR}/ghettoVCB-email-$$.header EMAIL_LOG_OUTPUT=${WORKDIR}/ghettoVCB-email-$$.log @@ -1405,13 +1622,21 @@ if [[ "${WORKDIR}" == "/" ]]; then exit 1 fi -if mkdir "${WORKDIR}"; then +if [[ -d "${WORKDIR}" ]] || mkdir "${WORKDIR}"; then #29.08.14 Added test "[[ -d ... ]] ||" + + # use of global ghettoVCB configuration #29.08.14 Moved here from sanityCheck because of "WORKDIR_DEBUG=1" and command below. + if [[ "${USE_GLOBAL_CONF}" -eq 1 ]] ; then + reConfigureGhettoVCBConfiguration "${GLOBAL_CONF}" + fi + + rm -rf ${WORKDIR}/* #29.08.14 Keep latest logs only, needs at least >${EMAIL_LOG_OUTPUT} in case WORKDIR is not deleted. + # create VM_FILE if we're backing up everything/specified a vm on the command line [[ $BACKUP_ALL_VMS -eq 1 ]] && touch ${VM_FILE} [[ -n "${VM_ARG}" ]] && echo "${VM_ARG}" > "${VM_FILE}" if [[ "${WORKDIR_DEBUG}" -eq 1 ]] ; then - LOG_TO_STDOUT=1 logger "info" "Workdir: ${WORKDIR} will not! be removed on exit" + LOG_TO_STDOUT=1 logger "info" "Workdir: ${WORKDIR} will not be removed on exit!" else # remove workdir when script finishes trap 'rm -rf "${WORKDIR}"' 0 @@ -1426,14 +1651,18 @@ if mkdir "${WORKDIR}"; then logger "info" "============================== ghettoVCB LOG START ==============================\n" logger "debug" "Succesfully acquired lock directory - ${WORKDIR}\n" - # terminate script and remove workdir when a signal is received - trap 'rm -rf "${WORKDIR}" ; exit 2' 1 2 3 13 15 + if [[ "${WORKDIR_DEBUG}" -eq 0 ]] ; then #29.08.14 Enclosed trap inside "if..;then...fi" + # terminate script and remove workdir when a signal is received + trap 'rm -rf "${WORKDIR}" ; exit 2' 1 2 3 13 15 + fi ghettoVCB ${VM_FILE} getFinalStatus - logger "debug" "Succesfully removed lock directory - ${WORKDIR}\n" + if [[ "${WORKDIR_DEBUG}" -eq 0 ]] ; then #16.10.14 + logger "debug" "Succesfully removed lock directory - ${WORKDIR}\n" + fi logger "info" "============================== ghettoVCB LOG END ================================\n" sendMail