diff --git a/README.md b/README.md index 0640998..33f3033 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,16 @@ or.. it is just a set of (Python) Wheels. Cloudify Plugins are packaged as sets of Python [Wheels](https://packaging.python.org/en/latest/distributing.html#wheels) in tar.gz archives and so we needed a tool to create such entities; hence, Wagon. -* Wagon currently supports Python 2.6.x and Python 2.7.x. + +* Wagon requires pip 1.4+ to work as this is the first version of pip to support Wheels. +* Wagon currently supports Python 2.6.x and Python 2.7.x. Python 2.5 will not be supported as it is not supported by pip. * Wagon is currently tested on both Linux and Windows (via Travis and AppVeyor). -* To be able to create Wagons of packages which include C extensions on Windows, you must have the [C++ Compiler for Python](http://www.microsoft.com/en-us/download/details.aspx?id=44266) installed. +* To be able to create Wagons of Wheels which include C extensions on Windows, you must have the [C++ Compiler for Python](http://www.microsoft.com/en-us/download/details.aspx?id=44266) installed. +* To be able to create Wagons of Wheels which include C extensions on Linux or OS X, you must have the required compiler installed depending on your base distro. Usually: + * RHEL based required `gcc` and `python-devel`. + * Debian based require `gcc` and `python-dev`. + * Arch-Linux requires `gcc`. + * OS X requires `gcc`. ## Installation @@ -38,12 +45,12 @@ wagon create --help wagon create -s flask # create an archive by retrieving the package from PyPI and keep the downloaded wheels (kept under /plugin) and exclude the cloudify-plugins-common and cloudify-rest-client packages from the archive. wagon create -s cloudify-script-plugin==1.2 --keep-wheels -v --exclude cloudify-plugins-common --exclude cloudify-rest-client -# create an archive by retrieving the source from a URL and creating wheels from requirement files found within the archive. Then, validation of the archive takes place. -wagon create -s http://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.tar.gz -r --validate +# create an archive by retrieving the source from a URL and creating wheels from requirement files found within the archive. Then, validation of the archive takes place. The created archive will be in zip format. +wagon create -s http://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.tar.gz -r --validate --format zip # create an archive by retrieving the source from a local path and output the tar.gz file to /tmp/.tar.gz (defaults to /.tar.gz) and provides explicit Python versions supported by the package (which usually defaults to the first two digits of the Python version used to create the archive.) wagon create -s ~/packages/cloudify-script-plugin/ -o /tmp/ --pyver 33 --pyver 26 --pyver 27 # pass additional args to `pip wheel` (NOTE that conflicting arguments are not handled by wagon.) -wagon create -s cloudify-script-plugin==1.2 -a '--retries 5' +wagon create -s http://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.zip -a '--retries 5' ``` #### Internal Requirement Files @@ -62,7 +69,6 @@ Wagon doesn't currently provide a way for packaging packages that are in editabl So, for instance, providing a dev-requirements file which contains a `-e DEPENDENCY` requirement will not be taken into consideration. This is not related to wagon but rather to the default `pip wheel` implementation stating that it will be "Skipping bdist_wheel for #PACKAGE#, due to being editable". We might allow processing editable provided dependencies in the future. - ### Install Packages ```shell @@ -73,11 +79,11 @@ wagon install --help ```shell # install a package from a local archive tar file and upgrade if already installed. Also, ignore the platform check which would force a package (whether it is or isn't compiled for a specific platform) to be installed. -wagon install -s ~/tars/cloudify_script_plugin-1.2-py27-none-any.tar.gz --upgrade --ignore-platform +wagon install -s ~/tars/cloudify_script_plugin-1.2-py27-none-any.wgn --upgrade --ignore-platform # install a package from a url into an existing virtualenv. -wagon install -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz --virtualenv my_venv -v +wagon install -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn --virtualenv my_venv -v # pass additional args to `pip install` (NOTE that conflicting arguments are not handled by wagon.) -wagon create -s cloudify-script-plugin==1.2 -a '--no-cache-dir' +wagon install -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn -a '--no-cache-dir' ``` Note that `--pre` is appended to the installation command to enable installation of prerelease versions. @@ -87,7 +93,7 @@ Note that `--pre` is appended to the installation command to enable installation While wagon provides a generic way of installing wagon created archives, you might not want to use the installer as you might not wish to install wagon on your application servers. Installing the package manually via pip is as easy as running (for example): ```shell -tar -xzvf http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz +tar -xzvf http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn pip install --no-index --find-links cloudify-script-plugin/wheels cloudify-script-plugin ``` @@ -108,9 +114,9 @@ Note that the `--validate` flag provided with the `create` function uses this sa ```shell # validate that an archive is a wagon compatible package -wagon validate -s ~/tars/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz +wagon validate -s ~/tars/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn # validate from a url -wagon validate -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz +wagon validate -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn ``` @@ -125,7 +131,7 @@ Given a Wagon archive, this will print its metadata. #### Examples ```shell -wagon showmeta -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz +wagon showmeta -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.wgn ``` @@ -149,7 +155,7 @@ A Metadata file is generated for the archive and looks somewhat like this: ``` { - "archive_name": "cloudify_script_plugin-1.2-py27-none-linux_x86_64-ubuntu-trusty.tar.gz", + "archive_name": "cloudify_script_plugin-1.2-py27-none-linux_x86_64-ubuntu-trusty.wgn", "build_server_os_properties": { "distribution": "ubuntu", "distribution_release": "trusty", @@ -192,7 +198,7 @@ A Metadata file is generated for the archive and looks somewhat like this: The archive is named according to the Wheel naming convention described in [PEP0491](https://www.python.org/dev/peps/pep-0491/#file-name-convention). -Example Output Archive: `cloudify_fabric_plugin-1.2.1-py27-none-any-none-none.tar.gz` +Example Output Archive: `cloudify_fabric_plugin-1.2.1-py27-none-any-none-none.wgn` * `{python tag}`: The Python version is set by the Python running the packaging process. That means that while a package might run on both py27 and py33 (for example), since the packaging process took place using Python 2.7, only py27 will be appended to the name. A user can also explicitly provide the supported Python versions for the package via the `pyver` flag. * `{platform tag}`: Normally, the platform (e.g. `linux_x86_64`, `win32`) is set for each specific wheel. To know which platform the package with its dependencies can be installed on, all wheels are checked. If a specific wheel has a platform property other than `any`, that platform will be used as the platform of the package. Of course, we assume that there can't be wheels downloaded or created on a specific machine platform that belongs to two different platforms. @@ -202,7 +208,7 @@ Example Output Archive: `cloudify_fabric_plugin-1.2.1-py27-none-any-none-none.ta ## Linux Support for compiled wheels -Example Output Archive: `cloudify_fabric_plugin-1.2.1-py27-none-linux_x86_64-ubuntu-trusty.tar.gz` +Example Output Archive: `cloudify_fabric_plugin-1.2.1-py27-none-linux_x86_64-ubuntu-trusty.wgn` Wheels which require compilation of C extensions and are compiled on Linux are not uploaded to PyPI due to variations between compilation environments on different distributions and links to varying system libraries. @@ -241,7 +247,7 @@ archive_path = w.create(with_requirements='', force=False, from wagon import wagon -source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.tar.gz' +source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.wgn' w = wagon.Wagon(source=source): w.install(virtualenv='', requirements_file='', upgrade=False, @@ -254,7 +260,7 @@ w.install(virtualenv='', requirements_file='', upgrade=False, from wagon import wagon -source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.tar.gz' +source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.wgn' w = wagon.Wagon(source=source): result = w.validate() # True if validation successful, else False @@ -266,7 +272,7 @@ result = w.validate() # True if validation successful, else False from wagon import wagon -source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.tar.gz' +source = 'http://my-wagons.com/flask-0.10.1-py27-none-linux_x86_64-Ubuntu-trusty.wgn' w = wagon.Wagon(source=source): metadata = w.get_metadata_from_archive() diff --git a/setup.py b/setup.py index f7e2a51..798de86 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ def read(*parts): setup( name='wagon', - version='0.2.7', + version='0.3.0', url='https://github.com/cloudify-cosmo/wagon', author='Gigaspaces', author_email='cosmo-admin@gigaspaces.com', diff --git a/wagon/tests/test_wagon.py b/wagon/tests/test_wagon.py index 392f324..6b10110 100644 --- a/wagon/tests/test_wagon.py +++ b/wagon/tests/test_wagon.py @@ -28,6 +28,7 @@ TEST_FILE = 'https://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.tar.gz' # NOQA +TEST_ZIP = 'https://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.zip' # NOQA TEST_PACKAGE_NAME = 'cloudify-script-plugin' TEST_PACKAGE_VERSION = '1.2' TEST_PACKAGE = '{0}=={1}'.format(TEST_PACKAGE_NAME, TEST_PACKAGE_VERSION) @@ -178,8 +179,12 @@ def tearDown(self): shutil.rmtree(self.package_name) def _test(self): - self.assertIn(self.archive_name, os.listdir('.')) - utils.untar(self.archive_name, '.') + # self.assertIn(self.archive_name, os.listdir('.')) + self.assertTrue(os.path.isfile(self.archive_name)) + try: + utils.untar(self.archive_name, '.') + except: + utils.unzip(self.archive_name, '.') with open(os.path.join( self.package_name, wagon.METADATA_FILE_NAME), 'r') as f: @@ -228,10 +233,24 @@ def test_create_archive_from_pypi_with_version(self): '-f': None } result = _invoke_click('create', params) - self.assertEqual(str(result), '') + self.assertIn('Process complete!', str(result.output)) m = self._test() self.assertEqual(m['package_source'], TEST_PACKAGE) + def test_create_zip_from_pypi(self): + self.archive_name = self.wagon.set_archive_name( + TEST_PACKAGE_NAME, TEST_PACKAGE_VERSION) + params = { + '-s': TEST_ZIP, + '-v': None, + '-f': None, + '-t': 'zip' + } + result = _invoke_click('create', params) + self.assertIn('Process complete!', str(result.output)) + m = self._test() + self.assertEqual(m['package_source'], TEST_ZIP) + def test_create_archive_from_pypi_with_additional_wheel_args(self): fd, reqs_file_path = tempfile.mkstemp() os.write(fd, 'virtualenv==13.1.2') @@ -243,7 +262,7 @@ def test_create_archive_from_pypi_with_additional_wheel_args(self): '--keep-wheels': None } result = _invoke_click('create', params) - self.assertEqual(str(result), '') + self.assertIn('Process complete!', str(result.output)) m = self._test() self.assertEqual(m['package_source'], TEST_PACKAGE) self.assertIn('virtualenv-13.1.2-py2.py3-none-any.whl', m['wheels']) @@ -276,7 +295,7 @@ def test_create_archive_from_url_with_requirements(self): '-r': None } result = _invoke_click('create', params) - self.assertEqual(str(result), '') + self.assertIn('Process complete!', str(result.output)) m = self._test() self.assertEqual(m['package_source'], TEST_FILE) @@ -301,7 +320,7 @@ def test_create_archive_with_exclusion(self): '-x': self.excluded_package } result = _invoke_click('create', params) - self.assertEqual(str(result), '') + self.assertIn('Process complete!', str(result.output)) m = self._test() self.assertEqual(len(m['excluded_wheels']), 1) @@ -314,7 +333,7 @@ def test_create_archive_with_missing_exclusion(self): '-x': self.missing_excluded_package } result = _invoke_click('create', params) - self.assertEqual(str(result), '') + self.assertIn('Process complete!', str(result.output)) m = self._test() self.assertEqual(len(m['excluded_wheels']), 0) diff --git a/wagon/utils.py b/wagon/utils.py index c4aac27..d8c3b23 100644 --- a/wagon/utils.py +++ b/wagon/utils.py @@ -18,6 +18,7 @@ import re import urllib import tarfile +import zipfile import logging from threading import Thread import time @@ -198,8 +199,22 @@ def download_file(url, destination): f.retrieve(final_url, destination) +def zip(source, destination): + lgr.info('Creating zip archive: {0}...'.format(destination)) + with closing(zipfile.ZipFile(destination, 'w')) as zip: + for root, dirs, files in os.walk(source): + for f in files: + zip.write(os.path.join(root, f)) + + +def unzip(archive, destination): + lgr.debug('Extracting zip {0} to {1}...'.format(archive, destination)) + with closing(zipfile.ZipFile(archive, "r")) as zip: + zip.extractall(destination) + + def tar(source, destination): - lgr.info('Creating archive: {0}...'.format(destination)) + lgr.info('Creating tar.gz archive: {0}...'.format(destination)) with closing(tarfile.open(destination, "w:gz")) as tar: tar.add(source, arcname=os.path.basename(source)) @@ -207,7 +222,7 @@ def tar(source, destination): def untar(archive, destination): """Extracts files from an archive to a destination folder. """ - lgr.debug('Extracting {0} to {1}...'.format(archive, destination)) + lgr.debug('Extracting tar.gz {0} to {1}...'.format(archive, destination)) with closing(tarfile.open(name=archive)) as tar: files = [f for f in tar.getmembers()] tar.extractall(path=destination, members=files) diff --git a/wagon/wagon.py b/wagon/wagon.py index e235f5e..4fe1fa8 100644 --- a/wagon/wagon.py +++ b/wagon/wagon.py @@ -62,7 +62,7 @@ def __init__(self, source, verbose=False): def create(self, with_requirements='', force=False, keep_wheels=False, excluded_packages=None, archive_destination_dir='.', python_versions=None, - validate=False, wheel_args=''): + validate=False, wheel_args='', format='tar.gz'): """Creates a Wagon archive and returns its path. This currently only creates tar.gz archives. The `install` @@ -126,7 +126,13 @@ def create(self, with_requirements='', force=False, self.handle_output_file(archive_path, force) self.generate_metadata_file(wheels, excluded_wheels) - utils.tar(package_name, archive_path) + if format == 'tar.gz': + utils.tar(package_name, archive_path) + elif format == 'zip': + utils.zip(package_name, archive_path) + else: + sys.exit('Unsupported archive format to create ' + '(Must be one of [zip, tar.gz]).') if not keep_wheels: lgr.debug('Cleaning up...') @@ -318,7 +324,7 @@ def set_archive_name(self, package_name, package_version): if release: archive[6] = release - self.archive = '{0}.tar.gz'.format('-'.join(archive)) + self.archive = '{0}.wgn'.format('-'.join(archive)) return self.archive def get_source(self, source): @@ -333,9 +339,17 @@ def get_source(self, source): that the string is a name of a package in PyPI. """ def extract_source(source, destination): - utils.untar(source, destination) - return os.path.join( + try: + utils.untar(source, destination) + except: + utils.unzip(source, destination) + source = os.path.join( destination, [d for d in os.walk(destination).next()[1]][0]) + if not os.path.join(source, 'setup.py'): + sys.exit('Source does not seem to be a Python package. ' + 'A source archive must contain a single parent ' + 'directory containing a setup.py file.') + return source self.remove_source_after_process = False @@ -431,6 +445,9 @@ def main(): help='Source URL, Path or Package name.') @click.option('-r', '--with-requirements', required=False, is_flag=True, help='Whether to also pack wheels from a requirements file.') +@click.option('-t', '--format', required=False, default='tar.gz', + type=click.Choice(['tar.gz', 'zip']), + help='Which file format to generate.') @click.option('-f', '--force', default=False, is_flag=True, help='Force overwriting existing output file.') @click.option('--keep-wheels', default=False, is_flag=True, @@ -449,7 +466,7 @@ def main(): help='Allows to pass additional arguments to `pip wheel`. ' '(e.g. --no-cache-dir -c constains.txt') @click.option('-v', '--verbose', default=False, is_flag=True) -def create(source, with_requirements, force, keep_wheels, exclude, +def create(source, with_requirements, format, force, keep_wheels, exclude, output_directory, pyver, validate, wheel_args, verbose): """Creates a Python package's wheel base archive. @@ -477,7 +494,7 @@ def create(source, with_requirements, force, keep_wheels, exclude, packager = Wagon(source, verbose) packager.create( with_requirements, force, keep_wheels, exclude, output_directory, - pyver, validate, wheel_args) + pyver, validate, wheel_args, format) @click.command()