From eec48eef1185f961a7a2b365d5767da397ec6678 Mon Sep 17 00:00:00 2001 From: nir0s Date: Thu, 22 Oct 2015 12:46:11 +0300 Subject: [PATCH 1/2] fixed windows related bugs and added appveyor config --- CHANGELOG | 44 --------------------------------------- Makefile | 7 +------ README.md | 4 ++-- TODO.md | 5 ----- appveyor.yml | 40 +++++++++++++++++++++++++++++++++++ setup.py | 2 +- tox.ini | 7 +++++++ wagon/tests/test_wagon.py | 31 ++++++++++++++++++++------- wagon/utils.py | 16 ++++++++++---- wagon/wagon.py | 8 ++++--- 10 files changed, 92 insertions(+), 72 deletions(-) delete mode 100644 CHANGELOG delete mode 100644 TODO.md create mode 100644 appveyor.yml diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 13335fd..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,44 +0,0 @@ -f67b4bf (HEAD, tag: 0.2.3, origin/master, origin/HEAD, master, allow-passing-arbitrary-pip-arguments) Merge pull request #9 from cloudify-cosmo/fix-requirement-files-not-being-dealt-with-correctly -8d87e55 (origin/fix-requirement-files-not-being-dealt-with-correctly, fix-requirement-files-not-being-dealt-with-correctly) fixed failure when trying to wheel a package that has dependencies in a requirements file -5e92335 (tag: 0.2.2) Merge pull request #7 from cloudify-cosmo/logger-fix -7d60140 (origin/logger-fix, logger-fix) changed module to package and now no longer overriding user's logger -21f000c (tag: 0.2.1) Update README.md -e8b78eb Update README.md -0a8a55d Update README.md -95b384b Merge pull request #6 from cloudify-cosmo/CFY-3709-add-module-exclusion -0adc2fa (origin/CFY-3709-add-module-exclusion, CFY-3709-add-module-exclusion) CFY-3709 added module exclusion, fixed a bug in validation and added showmeta cli func -c083124 Update README.md -6391892 Merge pull request #5 from cloudify-cosmo/refine-meta-properties -f9f5691 (origin/refine-meta-properties, refine-meta-properties) Added support for user provided python versions, ignoring the supported platform check and linux specific handling. -5fd3336 (tag: 0.2.0, tag: 0.1.2) Merge pull request #4 from cloudify-cosmo/add-installation-validation -673b72e moved from wheelr to wagon and added installation and creation validation -e2bf0b3 added more metadata info -35f3b2d docs update -ed8c6b9 docs update -8303ae1 (tag: 0.1) First release -ad48881 more fixes -9fc7bc3 test fixes -f6119a7 more tests -e33e157 doc fiX -80c6a55 doc fiX -a5189fb doc fiX -e7fc5d2 Merge pull request #3 from cloudify-cosmo/make-module-generic -b7a5029 validator -6f616c7 Merge pull request #2 from cloudify-cosmo/make-module-generic -52056fd Merge branch 'master' into make-module-generic -a0ce9a6 yay! -025ee77 Merge pull request #1 from cloudify-cosmo/logging_format -80f755a added upgrade supportg -4c82e3c now supporting installation of the module from the tar file -e1a31af made generic and added support for requirement files -a2c0141 many more tests -953d991 added tests for source path and source url -e0003e9 Merge branch 'master' of github.com:cloudify-cosmo/cloudify-plugin-packager -0cf7402 testing -c05a008 Merge branch 'master' into logging_format -9a6fe82 fix flake8 -7c420d1 strip() log lines from whitespaces and newlines -9638396 fix help text layout -3da1dd8 nothing -ed5ce71 error handling -abfe4dd CFY-3466-implemented-plugin-packager-utility diff --git a/Makefile b/Makefile index 9a62841..81259b3 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,12 @@ all: @echo "make dev - prepares a development environment (includes tests)" @echo "make instdev - prepares a development environment (no tests)" @echo "make install - install on local system" - @echo "make files - update changelog and todo files" @echo "make test - run tox" @echo "make docs - build docs" @echo "prepare - prepare module for release (CURRENTLY IRRELEVANT)" @echo "make publish - upload to pypi" -release: files docs publish +release: docs publish dev: instdev test @@ -22,10 +21,6 @@ instdev: install: python setup.py install -files: - grep '# TODO' -rn * --exclude-dir=docs --exclude-dir=build --exclude=TODO.md | sed 's/: \+#/: # /g;s/:#/: # /g' | sed -e 's/^/- /' | grep -v Makefile > TODO.md - git log --oneline --decorate --color > CHANGELOG - test: pip install tox tox diff --git a/README.md b/README.md index 26015c6..076121d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ or.. it is just a set of (Python) Wheels. Cloudify Plugins are packaged as sets of Python [Wheels](https://packaging.python.org/en/latest/distributing.html#wheels) in tar.gz archives and so we needed a tool to create such entities; hence, Wagon. * Wagon currently supports Python 2.6.x and Python 2.7.x. -* Wagon is currently only tested on Linux but might work on other platforms. +* Wagon is currently tested on both Linux and Windows (via Travis and AppVeyor). +* To be able to create Wagons of Wheels which include C extensions, you must have the [C++ Compiler for Python](http://www.microsoft.com/en-us/download/details.aspx?id=44266) installed. ## Installation @@ -216,4 +217,3 @@ tox * Support Python 3.x * Provide the most statistically robust way of identification and installation of Linux compiled Wheels. -* Test on Windows (AppVeyor to come...) diff --git a/TODO.md b/TODO.md deleted file mode 100644 index bca32e5..0000000 --- a/TODO.md +++ /dev/null @@ -1,5 +0,0 @@ -- wagon/wagon.py:361: # TODO: maybe we don't want to be that explicit and allow using >= -- wagon/wagon.py:362: # TODO: or just a package name... -- wagon/wagon.py:440: # TODO: Let the user provide supported Python versions. -- wagon/wagon.py:441: # TODO: Let the user provide supported Architectures. -- wagon/utils.py:48: # TODO: implement using sh diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..286e05e --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,40 @@ +environment: + + TOX_ENV: pywin + + matrix: + - PYTHON: C:\Python27 + PYTHON_VERSION: 2.7.8 + PYTHON_ARCH: 32 + +install: + + ################################# + # Change Python Registry + ################################# + + - reg ADD HKCU\Software\Python\PythonCore\2.7\InstallPath /ve /d "C:\Python27" /t REG_SZ /f + - reg ADD HKLM\Software\Python\PythonCore\2.7\InstallPath /ve /d "C:\Python27" /t REG_SZ /f + + ################################# + # Installing Inno Setup + ################################# + + - choco install -y InnoSetup + - set PATH="C:\\Program Files (x86)\\Inno Setup 5";%PATH% + + - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% + + - echo Upgrading pip... + - ps: (new-object System.Net.WebClient).Downloadfile('https://bootstrap.pypa.io/get-pip.py', 'C:\Users\appveyor\get-pip.py') + - ps: Start-Process -FilePath "C:\Python27\python.exe" -ArgumentList "C:\Users\appveyor\get-pip.py" -Wait -Passthru + - pip --version + +build: false # Not a C# project, build stuff at the test step instead. + +before_test: + - echo Installing tox (2.0.0) + - pip install tox==2.0.0 + +test_script: + - tox -e %TOX_ENV% \ No newline at end of file diff --git a/setup.py b/setup.py index c321ff2..5ed5bef 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def read(*parts): setup( name='wagon', - version='0.2.4', + version='0.2.5', url='https://github.com/cloudify-cosmo/wagon', author='Gigaspaces', author_email='cosmo-admin@gigaspaces.com', diff --git a/tox.ini b/tox.ini index 6582e15..c136407 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,13 @@ deps = -rdev-requirements.txt commands=nosetests --with-cov --cov-report term-missing --cov wagon wagon/tests -v +[testenv:pywin] +deps = + -rdev-requirements.txt +commands=nosetests -v --nocapture --nologcapture --with-cov --cov-report term-missing --cov wagon wagon/tests +basepython = {env:PYTHON:}\python.exe +passenv=ProgramFiles APPVEYOR LOGNAME USER LNAME USERNAME HOME USERPROFILE + [testenv:flake8] deps = flake8 diff --git a/wagon/tests/test_wagon.py b/wagon/tests/test_wagon.py index 9d397dd..d5b77a3 100644 --- a/wagon/tests/test_wagon.py +++ b/wagon/tests/test_wagon.py @@ -55,7 +55,7 @@ def test_run(self): def test_run_bad_command(self): p = utils.run('suname') - self.assertEqual(127, p.returncode) + self.assertEqual(1 if utils.IS_WIN else 127, p.returncode) def test_download_file(self): utils.download_file(TEST_FILE, 'file') @@ -72,7 +72,11 @@ def test_download_file_missing(self): def test_download_bad_url(self): e = self.assertRaises( IOError, utils.download_file, 'something', 'file') - self.assertIn("No such file or directory: 'something'", str(e)) + if utils.IS_WIN: + self.assertIn("The system cannot find the file " + "specified: 'something'", str(e)) + else: + self.assertIn("No such file or directory: 'something'", str(e)) def test_download_missing_path(self): e = self.assertRaises( @@ -80,6 +84,8 @@ def test_download_missing_path(self): self.assertIn('No such file or directory', e) def test_download_no_permissions(self): + if utils.IS_WIN: + self.skipTest('Irrelevant on Windows.') e = self.assertRaises(IOError, utils.download_file, TEST_FILE, '/file') self.assertIn('Permission denied', str(e)) @@ -96,6 +102,8 @@ def test_tar(self): os.remove('tar.file') def test_tar_no_permissions(self): + if utils.IS_WIN: + self.skipTest("Irrelevant on Windows.") tmpdir = tempfile.mkdtemp() try: e = self.assertRaises(IOError, utils.tar, tmpdir, '/file') @@ -105,7 +113,11 @@ def test_tar_no_permissions(self): def test_tar_missing_source(self): e = self.assertRaises(OSError, utils.tar, 'missing', 'file') - self.assertIn("No such file or directory: 'missing'", str(e)) + if utils.IS_WIN: + self.assertIn("The system cannot find the " + "file specified: 'missing'", str(e)) + else: + self.assertIn("No such file or directory: 'missing'", str(e)) os.remove('file') def test_wheel_nonexisting_package(self): @@ -116,9 +128,11 @@ def test_wheel_nonexisting_package(self): finally: shutil.rmtree('package') - # non-windows def test_machine_platform(self): - self.assertEqual(utils.get_machine_platform(), 'linux_x86_64') + if utils.IS_WIN: + self.assertIn('win', utils.get_machine_platform().lower()) + else: + self.assertEqual(utils.get_machine_platform(), 'linux_x86_64') class TestCreateBadSources(testtools.TestCase): @@ -157,7 +171,7 @@ def tearDown(self): shutil.rmtree(TEST_PACKAGE_NAME) def _test(self): - self.assertTrue(os.path.isfile(self.archive_name)) + # self.assertTrue(os.path.isfile(self.archive_name)) utils.untar(self.archive_name, '.') with open(os.path.join( TEST_PACKAGE_NAME, @@ -228,7 +242,10 @@ def test_create_archive_from_pypi_with_additional_wheel_args(self): os.close(fd) def test_create_archive_from_url_with_requirements(self): - self.wagon.platform = utils.get_machine_platform() + if utils.IS_WIN: + self.wagon.platform = 'win32' + else: + self.wagon.platform = utils.get_machine_platform() self.archive_name = self.wagon.set_archive_name( TEST_PACKAGE_NAME, TEST_PACKAGE_VERSION) params = { diff --git a/wagon/utils.py b/wagon/utils.py index 5267b17..725c306 100644 --- a/wagon/utils.py +++ b/wagon/utils.py @@ -17,7 +17,7 @@ IS_VIRTUALENV = hasattr(sys, 'real_prefix') PLATFORM = sys.platform -IS_WIN = (PLATFORM == 'win32') +IS_WIN = (os.name == 'nt') IS_DARWIN = (PLATFORM == 'darwin') IS_LINUX = (PLATFORM == 'linux2') @@ -224,7 +224,15 @@ def get_python_version(): def get_machine_platform(): - id = '{0}_{1}'.format(platform.system().lower(), platform.machine()) + system = platform.system().lower() + machine = platform.machine().lower() + id = 'unidentified' + if system and machine: + id = '{0}_{1}'.format(system, machine) + elif system: + id = system + elif machine: + id = machine lgr.info('Identified machine platform: {0}'.format(id)) return id @@ -261,11 +269,11 @@ def check_installed(package, virtualenv): return False -def make_virtualenv(virtualenv_dir, python_path='python'): +def make_virtualenv(virtualenv_dir): """This will create a virtualenv. """ lgr.debug('Creating Virtualenv {0}...'.format(virtualenv_dir)) - result = run('virtualenv -p {0} {1}'.format(python_path, virtualenv_dir)) + result = run('virtualenv {0}'.format(virtualenv_dir)) if not result.returncode == 0: lgr.error('Could not create virtualenv: {0}'.format(virtualenv_dir)) sys.exit(codes.errors['failed_to_create_virtualenv']) diff --git a/wagon/wagon.py b/wagon/wagon.py index ee21d13..85adcde 100644 --- a/wagon/wagon.py +++ b/wagon/wagon.py @@ -355,9 +355,9 @@ def get_source_name_and_version(self, source): lgr.debug('setup.py file found. Retrieving name and version...') setuppy_path = os.path.join(source, 'setup.py') self.name = utils.run('python {0} --name'.format( - setuppy_path)).aggr_stdout.strip('\n') + setuppy_path)).aggr_stdout.rstrip('\r\n') self.version = utils.run('python {0} --version'.format( - setuppy_path)).aggr_stdout.strip('\n') + setuppy_path)).aggr_stdout.rstrip('\r\n') # TODO: maybe we don't want to be that explicit and allow using >= # TODO: or just a package name... elif '==' in source: @@ -384,7 +384,9 @@ def handle_output_file(self, archive, force): def handle_output_directory(self, wheels_path, force): if os.path.isdir(wheels_path): if force: - shutil.rmtree(wheels_path) + # Ignore errors is currently a workaround for shutil failing + # on Windows when the directory is not empty. + shutil.rmtree(wheels_path, ignore_errors=True) else: lgr.error('Directory {0} already exists. Please remove it and ' 'run this again.'.format(wheels_path)) From 2010cba9cc048892f827893659d56d3af24f214d Mon Sep 17 00:00:00 2001 From: nir0s Date: Thu, 22 Oct 2015 18:24:21 +0300 Subject: [PATCH 2/2] fix platform --- wagon/tests/test_wagon.py | 8 ++++---- wagon/utils.py | 16 ++++------------ wagon/wagon.py | 2 +- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/wagon/tests/test_wagon.py b/wagon/tests/test_wagon.py index d5b77a3..4777f00 100644 --- a/wagon/tests/test_wagon.py +++ b/wagon/tests/test_wagon.py @@ -130,9 +130,9 @@ def test_wheel_nonexisting_package(self): def test_machine_platform(self): if utils.IS_WIN: - self.assertIn('win', utils.get_machine_platform().lower()) + self.assertIn('win32', utils.get_platform()) else: - self.assertEqual(utils.get_machine_platform(), 'linux_x86_64') + self.assertEqual(utils.get_platform(), 'linux_x86_64') class TestCreateBadSources(testtools.TestCase): @@ -171,7 +171,7 @@ def tearDown(self): shutil.rmtree(TEST_PACKAGE_NAME) def _test(self): - # self.assertTrue(os.path.isfile(self.archive_name)) + self.assertTrue(os.path.isfile(self.archive_name)) utils.untar(self.archive_name, '.') with open(os.path.join( TEST_PACKAGE_NAME, @@ -245,7 +245,7 @@ def test_create_archive_from_url_with_requirements(self): if utils.IS_WIN: self.wagon.platform = 'win32' else: - self.wagon.platform = utils.get_machine_platform() + self.wagon.platform = utils.get_platform() self.archive_name = self.wagon.set_archive_name( TEST_PACKAGE_NAME, TEST_PACKAGE_VERSION) params = { diff --git a/wagon/utils.py b/wagon/utils.py index 725c306..c7111bc 100644 --- a/wagon/utils.py +++ b/wagon/utils.py @@ -10,6 +10,8 @@ from contextlib import closing import platform +from wheel import pep425tags as wheel_tags + import codes import logger @@ -223,18 +225,8 @@ def get_python_version(): return 'py{0}{1}'.format(version[0], version[1]) -def get_machine_platform(): - system = platform.system().lower() - machine = platform.machine().lower() - id = 'unidentified' - if system and machine: - id = '{0}_{1}'.format(system, machine) - elif system: - id = system - elif machine: - id = machine - lgr.info('Identified machine platform: {0}'.format(id)) - return id +def get_platform(): + return wheel_tags.get_platform() def get_os_properties(): diff --git a/wagon/wagon.py b/wagon/wagon.py index 85adcde..8138c1e 100644 --- a/wagon/wagon.py +++ b/wagon/wagon.py @@ -141,7 +141,7 @@ def install(self, virtualenv=None, requirements_file=None, upgrade=False, if not ignore_platform and supported_platform != 'any': lgr.debug('Validating Platform {0} is supported...'.format( supported_platform)) - machine_platform = utils.get_machine_platform() + machine_platform = utils.get_platform() if machine_platform != supported_platform: lgr.error('Platform unsupported for package ({0}).'.format( machine_platform))