diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1073859 --- /dev/null +++ b/.gitignore @@ -0,0 +1,60 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#OVA images +*.ova diff --git a/changelog.txt b/changelog.txt new file mode 100644 index 0000000..2137e7e --- /dev/null +++ b/changelog.txt @@ -0,0 +1,9 @@ +0.1 01/29/2015 +- Initial release +- Support for creating local repos for offline updates + +0.2 04/07/2015 +- VURA now creates ISO images for offline updates +- Documentation improvements +- Updated system packages +- Improved ability to continue downloads diff --git a/old-version/readme.txt b/old-version/readme.txt new file mode 100644 index 0000000..98fb871 --- /dev/null +++ b/old-version/readme.txt @@ -0,0 +1,33 @@ +About + +The VAMI Update Repository Appliance is a tool to help you create a portable update repository for VAMI-enabled VMware appliances. This allows you to transport the necessary bits to perform upgrades to network locations without internet access. + +Prerequisites: +* VMware ESXi (www.vmware.com) +* VMware vSphere client (www.vmware.com) +* 7-zip (www.7-zip.org) + +Usage + +0. Extract vura.7z on a machine that can deploy VMs in your environment. +1. Deploy the appliance image on a host and network with internet access. +2. Configure the networking settings with SSH/Console access, if desired. The root password is "vura" by default. The system is Ubuntu in case you need to make adjustments that are not covered here. +3. Expand the /data volume of this appliance by adding a disk and rebooting or using LVM. You can verify that the disk space has been added by typing "df -h" on the command line of the appliance. +4. Navigate to the VAMI page of your desired appliance and locate the RepositoryURL on the Update>Settings tab. The page can usually be accessed by navigating to https://[Appliance IP Address]:5480. +5. Copy the RepositoryURL from the VAMI appliance to the URL field on the web page of your vura appliance. +6. Enter a repository name to help you identify the product name and version. No spaces, please. +7. Press the Create button. +8. Wait until the status under Current Repositories shows "Ready". +9. Move this appliance to your desired network location. +10a. Copy the URL of your new repo from the VURA status page into the VAMI Update page of your appliance. +10b. Download the ISO image and attach it to your VM. The Update page +11. Update and enjoy! + +Dependencies for running source version: +*Linux +*Nginx +*fcgiwrap +*python2 +*coreutils +*aria2c +*genisoimage diff --git a/old-version/vura-0.1-src.tar.gz b/old-version/vura-0.1-src.tar.gz new file mode 100644 index 0000000..67615f3 Binary files /dev/null and b/old-version/vura-0.1-src.tar.gz differ diff --git a/old-version/vura-0.2-src.tar.gz b/old-version/vura-0.2-src.tar.gz new file mode 100644 index 0000000..dcaea21 Binary files /dev/null and b/old-version/vura-0.2-src.tar.gz differ diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..601ea86 --- /dev/null +++ b/readme.md @@ -0,0 +1,44 @@ +## VURA: The VAMI Update Repository Appliance ## + +About: +----- + +The VAMI Update Repository Appliance is a tool to help you create a portable update repository for VAMI-enabled VMware appliances. This allows you to transport the necessary bits to perform upgrades to network locations without internet access. It can also help to generate update ISO images for appliances that do not have them available for download. + +Requirements: +---- + +[Source Version] + - Python 2.7 + - Internet access for initial repository creation + - User permission to bind port 80 + - (Optional) genisoimage - If you wish to be able to generate ISO images + +[vApp Version] + - ESXi 4.0U2+ + - 1 vCPU + - 512 MB RAM + - 16 GB Disk Space (thick provisioned) + - Internet access for initial repository creation + +Instructions +---- +[vApp Version] (recommended) +1. Deploy vApp in location with internet access +2. Obtain the Update URL from the administration page of your appliance (usually https:// [your appliance] :5480) +3. Point your browser to the ip address of the VURA appliance +4. Paste the URL in the URL field of the UI +5. Give the repository a name without spaces or special characters +6. Press the Create button +7. You may now either download the ISO image and attach it to your VM to upgrade it, or paste the Update URL into the administration page of your appliance + +[Source Version] +0. Ensure you have adequate disk space to host 2x the size of the update repository you will be downloading. You can assume it will be similar in size to the original OVA image you used to deploy it. +1. Download source tree to a machine that has Python 2.7 +2. Execute vura.py as a user that can bind port 80 +3. Follow appliance instructions starting at Step 2 + +Technologies Used: + - Python 2.7 + - CherryPy + - genisoimage \ No newline at end of file diff --git a/source/.index.html.~37b10bfc b/source/.index.html.~37b10bfc new file mode 100644 index 0000000..bcbf7be --- /dev/null +++ b/source/.index.html.~37b10bfc @@ -0,0 +1,92 @@ + + +VAMI Update Repository Appliance + + + + +

Welcome to the VAMI Update Repository Appliance

+
+

About

+ The VAMI Update Repository Appliance is a tool to help you create a portable update repository for VAMI-enabled VMware appliances. This allows you to transport the necessary bits to perform upgrades to network locations without internet access. +

Usage

+
    +
  1. Configure the networking settings with SSH/Console access, if desired. The root password is "vura" by default.
    The system is Ubuntu-based in case you need to make adjustments that are not covered here.
    2. +
  2. Expand the /data volume of this appliance by adding a disk and rebooting or using LVM.
    You can verify that the disk space has been added by typing "df -h" on the command line of the appliance.
    3. +
  3. Navigate to the VAMI page of your desired appliance and locate the RepositoryURL on the Update>Settings tab.
    The page can usually be accessed by navigating to https://[Appliance IP Address]:5480.
    4. +
  4. Copy the RepositoryURL to the URL field below.
    5. +
  5. Enter a repository name to help you identify the product name and version. No spaces, please.
    6. +
  6. Press the Create button.
    7. +
  7. Wait until the status under Current Repositories shows "Ready".
    8. +
  8. Move this appliance to your desired network location.
    9. +
  9. Copy the URL of your new repo into the VAMI Update page of your appliance.
    10. +
  10. Update and enjoy!
    11. +
+ Note: You can also download the update ISO and attach it to your VM to use as an update source, if you prefer. +
+

Create new repository:

+ + + +
+ +

Current repositories: +
+ + +
Repo NameURLISOSizeStatusAction
+ + + \ No newline at end of file diff --git a/source/launcher.sh b/source/launcher.sh new file mode 100755 index 0000000..92a693d --- /dev/null +++ b/source/launcher.sh @@ -0,0 +1,20 @@ +#!/bin/bash +trap exit SIGKILL SIGTERM SIGINT +echo $$ > vura.pid +if [ "$USER" != root ]; then + if [ -x "`which authbind`" ]; then + if [ ! -x "/etc/authbind/byport/80" ]; then + echo "authbind needs to be configured to allow unprivilidged use of port 80" + sudo touch /etc/authbind/byport/80 + sudo chown $USER:$USER /etc/authbind/byport/80 + sudo chmod 550 /etc/authbind/byport/80 + fi + authbind --deep python vura.py + else + echo "authbind not available, using sudo to bind port 80" + sudo python vura.py + fi +else + python vura.py +fi +rm vura.pid diff --git a/source/license.txt b/source/license.txt new file mode 100644 index 0000000..0e732e1 --- /dev/null +++ b/source/license.txt @@ -0,0 +1,24 @@ +Copyright (c) 2015, Jeremy McCoy +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of VMware, Jeremy McCoy nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/source/linux_supplements/diskadd.sh b/source/linux_supplements/diskadd.sh new file mode 100644 index 0000000..1de1768 --- /dev/null +++ b/source/linux_supplements/diskadd.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +VOLUME_GROUP=vg0 +VOLUME_NAME=data +MOUNT_PATH=/data +LOG="/var/log/diskadd.log" + +exitfunc() { + echo $1 | tee -a $LOG + #always try and mount the data disk + mount /dev/$VOLUME_GROUP/$VOLUME_NAME $MOUNT_PATH -o defaults,nodev,noacl,nosuid,exec | tee -a $LOG 2>&1 + exit 1 +} + +echo "Running diskadd at `date`" | tee -a $LOG + +#activate the logical volume +lvchange -ay /dev/$VOLUME_GROUP/$VOLUME_NAME | tee -a $LOG 2>&1 +if [ $? -ne 0 ]; then + exitfunc "Could not activate logical volume" +fi + +#identify all disks with no partitions (leave out /dev/dm-0 as that is not a disk we have added) +disks=(`fdisk -l 2>&1 | grep -v '/dev/dm-'| grep 'contain a valid partition table' | grep -v "/dev/mapper/$VOLUME_GROUP-" | awk '{print $2}' |tr '\n' ' '`) +#Get the list of LVM physical volumes +pvolumes=(`pvdisplay | grep 'PV Name' | awk '{print $3}'`) +expanded=false +if [ ${#disks[*]} -ne 0 ] +then + #unmount the data disk + umount /dev/$VOLUME_GROUP/$VOLUME_NAME | tee -a $LOG 2>&1 + if [ $? -ne 0 ]; then + exitfunc "Could not unmount data disk" + fi + + #for all disks with no partition check to see if there is a lvm physical volume with the same name, if not create one + for disk in ${disks[@]} + do + echo Processing $disk | tee -a $LOG + found=false; + for volume in ${pvolumes[@]} + do + if [ $disk == $volume ]; then + found=true; + break; + fi + done + # new disk with no LVM physical volume. Create one and add it to the volume group + if [ $found == false ]; then + echo "Adding new disk $disk to the LVM logical group" + echo "Adding new disk $disk to the LVM logical group" | tee -a $LOG + expanded=true; + # create a physical volume. + pvcreate $disk | tee -a $LOG 2>&1 + if [ $? -ne 0 ]; then + echo "Could not create Physical Volume" | tee -a $LOG + continue + fi + + vgextend $VOLUME_GROUP $disk | tee -a $LOG 2>&1 + if [ $? -ne 0 ]; then + echo "Could not extend Volume Group" | tee -a $LOG + continue + fi + extents=`pvdisplay $disk | grep "Free PE" | awk '{print $3}'` + lvextend -fnrl +${extents} /dev/$VOLUME_GROUP/$VOLUME_NAME | tee -a $LOG 2>&1 + if [ $? -ne 0 ]; then + echo "Could not extend Logical Volume" | tee -a $LOG + continue + fi + fi + done +fi + +#finally mount the data disk +mount -a 2>&1 | tee -a $LOG 2>&1 + +exit 0 diff --git a/source/linux_supplements/issue.sh b/source/linux_supplements/issue.sh new file mode 100644 index 0000000..79574d9 --- /dev/null +++ b/source/linux_supplements/issue.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ip="`ifconfig eth0 | grep 'inet addr' | cut -d ':' -f2 | awk '{ print $1 }'`" +echo "Welcome to the VAMI Update Repository Appliance! +To access the UI, please direct your browser to http://$ip/ or log in to configure. +The default root password is vura. + +`df -h`" | tee /etc/issue diff --git a/source/linux_supplements/readme.txt b/source/linux_supplements/readme.txt new file mode 100644 index 0000000..3462278 --- /dev/null +++ b/source/linux_supplements/readme.txt @@ -0,0 +1,17 @@ +The scripts in this directory will assist you in configuring your own VURA appliance. Please note that they were intended to be run on Ubuntu 14.04, and may need to be adapted to run on your system. + +If the partition VURA lives in is a logicial volume dedicated to it, the scripts under linux_suppliments should be copied to another partition to allow them to function properly. + +The following additions to rc.local are recommended: + + # Detect new disks and add them to the logical volume you store VURA on. This is optional, and requires configuration to match how you have set up your logical volumes. If you are not using logical volumes, please comment out this command. + bash /opt/diskadd.sh + sync + sleep 5 + mount -a + + # Set the login message to include the local IP address and login info. This is optional, and can be adjusted as you see fit. + bash /opt/issue.sh + + # Run VURA as the user 'vura'. If the VURA user does not have permissions to bind to port 80, you will need to install and configure authbind or run this as root. + su vura -c 'bash -c "cd /data/vura && ./launcher.sh &"' diff --git a/source/vura.py b/source/vura.py new file mode 100644 index 0000000..80c8cd9 --- /dev/null +++ b/source/vura.py @@ -0,0 +1,220 @@ +import cherrypy, string, shutil, os, os.path, glob, sys, thread, threading, urllib, concurrent, futures, socket, re, subprocess, distutils, mimetypes +from concurrent.futures.thread import ThreadPoolExecutor +from distutils import spawn + +# Get IP address +def getip(): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(('8.8.8.8', 0)) + local_ip_address = s.getsockname()[0] + return local_ip_address + except: + cherrypy.log('| ERROR | Unable determine IP address') + return "localhost" + +# Create thread for downloader +downloadThread = threading.Thread() + +# Create progress counter for downloader +progress = 0 + +# Check for genisoimage +genisoimage = distutils.spawn.find_executable('genisoimage') +if os.path.exists(genisoimage): + canimakeaniso = "True" +else: + canimakeaniso = "False" + +def get_folder_size(folder_path): +# Because I don't want to depend on a system call + try: + folder_size = 0 + for (path, dirs, files) in os.walk(folder_path): + for file in files: + filename = os.path.join(path, file) + folder_size += os.path.getsize(filename) + return str(folder_size/1024/1024) + 'MB' + except: + cherrypy.log('| ERROR | Unable to get folder size for '+name) + +# Function to write repo state file +def statewriter(name, message): + try: + with open('repo/'+name+'/state','w') as state: + state.write(message) + state.close() + except: + cherrypy.log('| ERROR | Unable to write state file for '+name) + +# Download individual file and increment progress counter +def fetch(name, url, destination, queuesize): + global progress + try: + #cherrypy.log('| INFO | Downloading '+destination) + urllib.urlretrieve(url, destination) + except: + cherrypy.log('| ERROR | Unable to download '+destination) + progress += 1 + statewriter(name, 'Downloaded '+str(progress)+'/'+str(queuesize)) + +# Queue multiple downloads and reset progress counter +def downloader(queue, name): + queuesize = len(queue) + global progress + progress = 0 + cherrypy.log("| INFO | Downloading "+str(queuesize)+" files for "+name) + with ThreadPoolExecutor(max_workers=8) as executor: + for url, destination in zip(queue.keys(), queue.values()): + executor.submit(fetch, name, url, destination, queuesize) + cherrypy.log("| INFO | Finished downloading "+str(queuesize)+" files for "+name) + +# Populate new repositories +def repoman(name): + try: + cherrypy.log("| INFO | Initializing repo: "+name) + statewriter(name, 'Initializing') + #read base url from file + baseurl=open('repo/'+name+'/url').read() + #build list of manifest files to download + manifest={} + manifest[baseurl+'/manifest/manifest-latest.xml']='repo/'+name+'/manifest/manifest-latest.xml' + manifest[baseurl+'/manifest/manifest-latest.xml.sha256']='repo/'+name+'/manifest/manifest-latest.xml.sha256' + manifest[baseurl+'/manifest/manifest-latest.xml.sig']='repo/'+name+'/manifest/manifest-latest.xml.sig' + manifest[baseurl+'/manifest/manifest-repo.xml']='repo/'+name+'/manifest/manifest-repo.xml' + except: + cherrypy.log("| ERROR | Initializing repo failed for "+name) + #download manifest files + downloader(manifest, name) + try: + cherrypy.log("| INFO | Building download list for "+name) + #read manifest file + manifest_file = open('repo/'+name+'/manifest/manifest-latest.xml') + #create dictionary of packages to download + package_pool={} + for line in manifest_file: + if re.match('package-pool', line): + pkg = line.rstrip() + package_pool[baseurl+'/'+pkg]='repo/'+name+'/'+pkg + except: + cherrypy.log("| ERROR | Building download list failed for "+name) + #download repo packages + downloader(package_pool, name) + statewriter(name, 'Ready') + #generate update.iso, if possible + if canimakeaniso is "True": + try: + cherrypy.log("| INFO | Building ISO for "+name) + os.popen(genisoimage+' -f -r -U -J -joliet-long -o update.iso.tmp repo/'+name).close() + shutil.move('update.iso.tmp','repo/'+name+'/update.iso') + except: + cherrypy.log("| ERROR | Building ISO failed for "+name) + else: + cherrypy.log("| WARN | genisoimage not installed, skipping ISO creation for "+name) + +# Initialize new repo +def createrepo(name, url): + global downloadThread + if not os.path.exists('repo/'+name): + cherrypy.log("| INFO | Creating repo: "+name) + os.makedirs('repo/'+name) + os.makedirs('repo/'+name+'/manifest') + os.makedirs('repo/'+name+'/package-pool') + else: + cherrypy.log("| WARN | Unable to create repo directory for "+name+", skipping") + statewriter(name, 'Starting') + #save repo source URL to file + try: + with open('repo/'+name+'/url','w') as urlfile: + urlfile.write(url) + urlfile.close() + except: + cherrypy.log('| ERROR | Unable to write URL file for '+name) + #begin downloading + if downloadThread.isAlive(): + cherrypy.log("| ERROR | Download already in progress! Skipping "+name) + else: + cherrypy.log("| INFO | Preparing Download") + downloadThread = threading.Thread(target=repoman, args=[name]) + downloadThread.start() + return """""" + +def deleterepo(name): + try: + cherrypy.log("| INFO | Deleting repo: "+name) + shutil.rmtree('repo/'+name) + except: + cherrypy.log("| ERROR | Deleting repo failed for "+name) + return """""" + +class ui(object): + #display web UI + @cherrypy.expose + def index(self): + return file('index.html') + + #List repos and stats. Formatted in JSON for DataTables + @cherrypy.expose + def list(self, _): + try: + #cherrypy.log('| INFO | Listing repos') + repocount = len(os.listdir('repo')) + jsondata='{"data":[' + for reponame in os.listdir('repo'): + if os.path.exists('repo/'+reponame+'/state'): + status = open('repo/'+reponame+'/state').read() + else: + status = 'Error' + cherrypy.log('| ERROR | Unable to read repo state file for '+reponame) + address = 'http://'+getip()+'/repo/'+reponame + if os.path.exists('repo/'+reponame+'/update.iso'): + isofile = 'Download ISO' + elif canimakeaniso is "False": + isofile = 'genisoimage not installed' + elif os.path.exists('update.iso.tmp'): + isofile = 'Not ready' + elif 'ownload' in status: + isofile = 'Not ready' + else: + isofile = 'Unavailable' + reposize = get_folder_size('repo/'+reponame) + deletebutton = '' + jsondata += '["'+reponame+'","'+address+'","'+isofile+'","'+reposize+'","'+status+'","'+deletebutton+'"]' + repocount -= 1 + if repocount > 0: + jsondata += ',' + jsondata += ']}' + return jsondata + except: + cherrypy.log('| ERROR | Listing repos failed') + + #receive creation request from web form and kick off thread + @cherrypy.expose + def create(self, create, name, url): + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(createrepo, name, url) + return """""" + + #delete repository + @cherrypy.expose + def delete(self, name): + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(deleterepo, name) + return """""" + +conf = { + 'global': { + 'log.access_file': "vura-access.log", + 'log.error_file': "vura-actions.log", + 'server.socket_host': '0.0.0.0', + 'server.socket_port': 80 + }, + '/': { + 'tools.staticdir.root': os.path.abspath(os.getcwd()) + }, + '/repo': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'repo' + } +} +cherrypy.quickstart(ui(), '/', conf)