From 15463208d8faba217950c2bb4049a520ebed4dce Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Tue, 20 Dec 2016 18:58:36 +0100 Subject: [PATCH 1/7] Fix deprecation warnings --- manage.py | 8 ++++---- spkrepo/app.py | 2 +- spkrepo/views/admin.py | 14 +++++++------- spkrepo/views/frontend.py | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manage.py b/manage.py index db3cd1b..111122e 100644 --- a/manage.py +++ b/manage.py @@ -7,11 +7,11 @@ from unittest import TextTestRunner from flask import current_app -from flask.ext.migrate import MigrateCommand -from flask.ext.script import Manager, Option -from flask.ext.security.script import (CreateUserCommand, DeactivateUserCommand, ActivateUserCommand, CreateRoleCommand, +from flask_migrate import MigrateCommand +from flask_script import Manager, Option +from flask_security.script import (CreateUserCommand, DeactivateUserCommand, ActivateUserCommand, CreateRoleCommand, RemoveRoleCommand, AddRoleCommand, commit) -from flask.ext.security.utils import encrypt_password +from flask_security.utils import encrypt_password from spkrepo import create_app from spkrepo.ext import db diff --git a/spkrepo/app.py b/spkrepo/app.py index 6cc70f6..3cc40e9 100644 --- a/spkrepo/app.py +++ b/spkrepo/app.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import jinja2 from flask import Flask -from flask.ext.admin import Admin +from flask_admin import Admin from wtforms import HiddenField from . import config as default_config diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index d1c9e5c..4c6b7de 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -4,13 +4,13 @@ import shutil from flask import flash, url_for, redirect, abort, current_app -from flask.ext.admin import AdminIndexView, expose -from flask.ext.admin.actions import action -from flask.ext.admin.contrib.sqla import ModelView -from flask.ext.admin.contrib.sqla.form import get_form -from flask.ext.admin.contrib.sqla.tools import get_query_for_ids -from flask.ext.admin.form import ImageUploadField -from flask.ext.security import current_user +from flask_admin import AdminIndexView, expose +from flask_admin.actions import action +from flask_admin.contrib.sqla import ModelView +from flask_admin.contrib.sqla.form import get_form +from flask_admin.contrib.sqla.tools import get_query_for_ids +from flask_admin.form import ImageUploadField +from flask_security import current_user from markupsafe import Markup from wtforms import PasswordField from wtforms.validators import Regexp diff --git a/spkrepo/views/frontend.py b/spkrepo/views/frontend.py index 694135c..a4e2e58 100644 --- a/spkrepo/views/frontend.py +++ b/spkrepo/views/frontend.py @@ -3,9 +3,9 @@ import os from flask import Blueprint, render_template, redirect, url_for, abort -from flask.ext.security import current_user, login_required, ConfirmRegisterForm -from flask.ext.security.forms import ChangePasswordForm -from flask.ext.wtf import Form +from flask_security import current_user, login_required, ConfirmRegisterForm +from flask_security.forms import ChangePasswordForm +from flask_wtf import FlaskForm from wtforms import ValidationError, StringField, SubmitField from wtforms.validators import InputRequired, Length @@ -21,7 +21,7 @@ def index(): return render_template('frontend/index.html') -class GenerateApiKeyForm(Form): +class GenerateApiKeyForm(FlaskForm): """Form for generating an API key""" api_key = StringField('API Key') submit = SubmitField('Generate API Key') From cdcce6c416861b17eb69f65d693a9de53dec52cb Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Tue, 20 Dec 2016 18:59:23 +0100 Subject: [PATCH 2/7] fake-factory has been renamed to Faker --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b86017..b3d705c 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ tests_require = ['Flask-Testing', - 'factory-boy', 'fake-factory', + 'factory-boy', 'Faker', 'lxml', 'urltools', 'coveralls'] From 2ffe8381b326150de33e6cd5ed49400664db2764 Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Tue, 20 Dec 2016 18:59:44 +0100 Subject: [PATCH 3/7] Remove unavailable requirement --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0f82f8d..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ --e git+https://github.com/Diaoul/flask-login.git@0.2.11-fix-session-creation#egg=Flask-Login -e . From 0e9a6539aeaaf57295419d3c78859d0353bad5ef Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Tue, 20 Dec 2016 19:00:07 +0100 Subject: [PATCH 4/7] Flask-Login 0.3.0 changed is_authenticated to property --- spkrepo/templates/layout.html | 2 +- spkrepo/views/admin.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spkrepo/templates/layout.html b/spkrepo/templates/layout.html index 477153c..80c2eca 100644 --- a/spkrepo/templates/layout.html +++ b/spkrepo/templates/layout.html @@ -48,7 +48,7 @@ {% endfor %} </ul> <ul class="nav navbar-nav navbar-right"> - {% if current_user.is_authenticated() %} + {% if current_user.is_authenticated %} {% if current_user.has_role('developer') or current_user.has_role('package_admin') or current_user.has_role('admin') %} {% set right_nav = [ diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 4c6b7de..2a40eeb 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -27,7 +27,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and current_user.has_role('admin') + return current_user.is_authenticated and current_user.has_role('admin') can_create = False @@ -73,7 +73,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and current_user.has_role('package_admin') + return current_user.is_authenticated and current_user.has_role('package_admin') can_edit = False @@ -87,7 +87,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and current_user.has_role('package_admin') + return current_user.is_authenticated and current_user.has_role('package_admin') can_edit = False @@ -121,7 +121,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and current_user.has_role('package_admin') + return current_user.is_authenticated and current_user.has_role('package_admin') # View def _display(view, context, model, name): @@ -144,7 +144,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and current_user.has_role('package_admin') + return current_user.is_authenticated and current_user.has_role('package_admin') @property def can_create(self): @@ -197,7 +197,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and any(map(current_user.has_role, ('developer', 'package_admin'))) + return current_user.is_authenticated and any(map(current_user.has_role, ('developer', 'package_admin'))) can_create = False @@ -372,7 +372,7 @@ def __init__(self, **kwargs): # Permissions def is_accessible(self): - return current_user.is_authenticated() and any(map(current_user.has_role, ('developer', 'package_admin'))) + return current_user.is_authenticated and any(map(current_user.has_role, ('developer', 'package_admin'))) can_create = False From 9f6b0f2548dafae37510cabe4be6c0148f45ba19 Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Tue, 20 Dec 2016 19:00:26 +0100 Subject: [PATCH 5/7] DSM6 support Add support for user creation and config file handling added in DSM6 --- migrations/versions/a41b2b645a3c_.py | 23 +++++++++++++++++++++++ spkrepo/models.py | 2 ++ spkrepo/tests/common.py | 28 ++++++++++++++++++++++++++-- spkrepo/tests/test_nas.py | 12 ++++++++++++ spkrepo/tests/test_utils.py | 18 ++++++++++++++++-- spkrepo/utils.py | 19 +++++++++++++++++++ spkrepo/views/api.py | 2 ++ spkrepo/views/nas.py | 4 ++++ 8 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/a41b2b645a3c_.py diff --git a/migrations/versions/a41b2b645a3c_.py b/migrations/versions/a41b2b645a3c_.py new file mode 100644 index 0000000..a819c70 --- /dev/null +++ b/migrations/versions/a41b2b645a3c_.py @@ -0,0 +1,23 @@ +"""Update for DSM6 + +Revision ID: a41b2b645a3c +Revises: 26b4c36c11e +Create Date: 2016-12-20 18:30:15.449680 + +""" +revision = 'a41b2b645a3c' +down_revision = '26b4c36c11e' + +from alembic import op +import sqlalchemy as sa + + + +def upgrade(): + op.add_column('version', sa.Column('conf_privilege', sa.Unicode(length=255), nullable=True)) + op.add_column('version', sa.Column('conf_resource', sa.Unicode(length=255), nullable=True)) + + +def downgrade(): + op.drop_column('version', 'conf_resource') + op.drop_column('version', 'conf_privilege') diff --git a/spkrepo/models.py b/spkrepo/models.py index e7d2ea7..964567a 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -363,6 +363,8 @@ class Version(db.Model): conf_dependencies = db.Column(db.Unicode(255)) conflicts = db.Column(db.Unicode(255)) conf_conflicts = db.Column(db.Unicode(255)) + conf_privilege = db.Column(db.Unicode(255)) + conf_resource = db.Column(db.Unicode(255)) install_wizard = db.Column(db.Boolean) upgrade_wizard = db.Column(db.Boolean) startable = db.Column(db.Boolean) diff --git a/spkrepo/tests/common.py b/spkrepo/tests/common.py index c7a5095..a2e7b4e 100644 --- a/spkrepo/tests/common.py +++ b/spkrepo/tests/common.py @@ -457,7 +457,7 @@ def create_image(text, width=640, height=480): def create_spk(build, info=None, signature=None, with_checksum=False, with_package_icons=True, with_info_icons=False, with_info=True, with_package=True, with_scripts=True, with_conf=False, info_encoding='utf-8', license_encoding='utf-8', signature_encoding='ascii', conf_dependencies_encoding='utf-8', - conf_conflicts_encoding='utf-8'): + conf_conflicts_encoding='utf-8', conf_privilege_encoding='utf-8', conf_resource_encoding='utf-8'): """ Create a valid SPK file @@ -478,6 +478,8 @@ def create_spk(build, info=None, signature=None, with_checksum=False, with_packa :param signature_encoding: encoding for the syno_signature.asc file :param conf_dependencies_encoding: encoding for the conf/PKG_DEPS file :param conf_conflicts_encoding: encoding for the conf/PKG_CONX file + :param conf_privilege_encoding: encoding for the conf/privilege file + :param conf_resource_encoding: encoding for the conf/resource file :return: the created SPK stream """ # generate an info if none is given @@ -506,7 +508,7 @@ def create_spk(build, info=None, signature=None, with_checksum=False, with_packa spk.addfile(signature_tarinfo, fileobj=signature_stream) # conf - if with_conf or build.version.conf_dependencies is not None or build.version.conf_conflicts is not None: + if with_conf or build.version.conf_dependencies is not None or build.version.conf_conflicts or build.version.conf_privilege is not None: conf_folder_tarinfo = tarfile.TarInfo('conf') conf_folder_tarinfo.type = tarfile.DIRTYPE conf_folder_tarinfo.mode = 0o755 @@ -533,6 +535,28 @@ def create_spk(build, info=None, signature=None, with_checksum=False, with_packa conf_tarinfo.size = conf_stream_bytes.tell() conf_stream_bytes.seek(0) spk.addfile(conf_tarinfo, fileobj=conf_stream_bytes) + if build.version.conf_privilege is not None: + conf_tarinfo = tarfile.TarInfo('conf/privilege') + config = ConfigParser() + config.read_dict(json.loads(build.version.conf_privilege)) + conf_stream = io.StringIO() + config.write(conf_stream) + conf_stream_bytes = io.BytesIO(conf_stream.getvalue().encode(conf_privilege_encoding)) + conf_stream_bytes.seek(0, io.SEEK_END) + conf_tarinfo.size = conf_stream_bytes.tell() + conf_stream_bytes.seek(0) + spk.addfile(conf_tarinfo, fileobj=conf_stream_bytes) + if build.version.conf_resource is not None: + conf_tarinfo = tarfile.TarInfo('conf/resource') + config = ConfigParser() + config.read_dict(json.loads(build.version.conf_resource)) + conf_stream = io.StringIO() + config.write(conf_stream) + conf_stream_bytes = io.BytesIO(conf_stream.getvalue().encode(conf_resource_encoding)) + conf_stream_bytes.seek(0, io.SEEK_END) + conf_tarinfo.size = conf_stream_bytes.tell() + conf_stream_bytes.seek(0) + spk.addfile(conf_tarinfo, fileobj=conf_stream_bytes) # wizards wizards = [] diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py index 0fab9a9..27c9731 100644 --- a/spkrepo/tests/test_nas.py +++ b/spkrepo/tests/test_nas.py @@ -110,6 +110,18 @@ def assertCatalogEntry(self, entry, build, data=None, link_params=None): else: self.assertNotIn('conf_conxpkgs', entry) + # conf_privilege + if build.version.conf_privilege: + self.assertEqual(entry['conf_privilege'], build.version.conf_privilege) + else: + self.assertNotIn('conf_privilege', entry) + + # conf_resource + if build.version.conf_resource: + self.assertEqual(entry['conf_resource'], build.version.conf_resource) + else: + self.assertNotIn('conf_resource', entry) + def test_missing_data_arch(self): self.assert400(self.client.post(url_for('nas.catalog'), data=dict(build='1594', language='enu'))) diff --git a/spkrepo/tests/test_utils.py b/spkrepo/tests/test_utils.py index 4f99301..8561ecd 100644 --- a/spkrepo/tests/test_utils.py +++ b/spkrepo/tests/test_utils.py @@ -41,7 +41,7 @@ def test_generic(self): self.assertEqual(build.version.maintainer_url, spk.info['maintainer_url']) self.assertEqual(build.version.package.name, spk.info['package']) self.assertEqual(build.version.report_url, spk.info['report_url']) - self.assertEqual(build.version.conf_dependencies is not None or build.version.conf_conflicts is not None, + self.assertEqual(build.version.conf_dependencies is not None or build.version.conf_conflicts is not None or build.version.conf_privilege is not None, spk.info['support_conf_folder']) self.assertEqual(build.version.version_string, spk.info['version']) @@ -185,7 +185,7 @@ def test_invalid_spk(self): self.assertEqual('Invalid SPK', str(cm.exception)) def test_missing_conf_folder(self): - build = BuildFactory.build(version__conf_dependencies=None, version__conf_conflicts=None) + build = BuildFactory.build(version__conf_dependencies=None, version__conf_conflicts=None, version__conf_privilege=None) info = create_info(build) info['support_conf_folder'] = 'yes' with create_spk(build, info=info, with_conf=False) as f: @@ -207,6 +207,20 @@ def test_wrong_conf_conflicts_encoding(self): SPK(f) self.assertEqual('Wrong conf/PKG_CONX encoding', str(cm.exception)) + def test_wrong_conf_privilege_encoding(self): + build = BuildFactory.build(version__conf_privilege=json.dumps({'déçu': {'run-as': '<run-as>'}})) + with create_spk(build, conf_privilege_encoding='latin-1') as f: + with self.assertRaises(SPKParseError) as cm: + SPK(f) + self.assertEqual('Wrong conf/privilege encoding', str(cm.exception)) + + def test_wrong_conf_resource_encoding(self): + build = BuildFactory.build(version__conf_resource=json.dumps({'déçu': {'<resource-id>': '<specification>'}})) + with create_spk(build, conf_resource_encoding='latin-1') as f: + with self.assertRaises(SPKParseError) as cm: + SPK(f) + self.assertEqual('Wrong conf/resource encoding', str(cm.exception)) + def test_empty_conf_folder(self): build = BuildFactory.build(version__conf_dependencies=None, version__conf_conflicts=None) info = create_info(build) diff --git a/spkrepo/utils.py b/spkrepo/utils.py index 5c41760..749569a 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -59,6 +59,8 @@ def __init__(self, stream): self.stream = stream self.conf_dependencies = None self.conf_conflicts = None + self.conf_privilege = None + self.conf_resource = None self.stream.seek(0) try: @@ -153,6 +155,23 @@ def __init__(self, stream): self.conf_conflicts = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) if self.conf_dependencies is None and self.conf_conflicts is None: raise SPKParseError('Empty conf folder') + if 'conf/privilege' in names: + c = ConfigParser() + try: + c.read_string(spk.extractfile('conf/privilege').read().decode('utf-8')) + except UnicodeDecodeError: + raise SPKParseError('Wrong conf/privilege encoding') + self.conf_privilege = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) + if 'conf/resource' in names: + c = ConfigParser() + try: + c.read_string(spk.extractfile('conf/resource').read().decode('utf-8')) + except UnicodeDecodeError: + raise SPKParseError('Wrong conf/resource encoding') + self.conf_resource = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) + if self.conf_dependencies is None and self.conf_conflicts is None and \ + self.conf_privilege is None and self.conf_resource is None: + raise SPKParseError('Empty conf folder') # verify checksum if 'checksum' in self.info: diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py index 8b0a96c..165058d 100644 --- a/spkrepo/views/api.py +++ b/spkrepo/views/api.py @@ -136,6 +136,8 @@ def post(self): conf_dependencies=spk.conf_dependencies, conflicts=spk.info.get('install_conflict_packages'), conf_conflicts=spk.conf_conflicts, + conf_privilege=spk.conf_privilege, + conf_resource=spk.conf_resource, install_wizard='install' in spk.wizards, upgrade_wizard='upgrade' in spk.wizards, startable=spk.info.get('startable'), license=spk.license) diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index 9bdd380..e1feafc 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -106,6 +106,10 @@ def get_catalog(arch, build, language, beta): entry['conf_deppkgs'] = b.version.conf_dependencies if b.version.conf_conflicts: entry['conf_conxpkgs'] = b.version.conf_conflicts + if b.version.conf_privilege: + entry['conf_privilege'] = b.version.conf_privilege + if b.version.conf_resource: + entry['conf_resource'] = b.version.conf_resource entries.append(entry) # DSM 5.1 From 6a7711b8e15d20e90b53e67bc95b42fb0617ff37 Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Wed, 21 Dec 2016 12:41:00 +0100 Subject: [PATCH 6/7] Fix processing of conf dir - Remove duplicate line - Just make sure we can read privilege and resource files in UTF-8, no further processing needed. --- spkrepo/utils.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/spkrepo/utils.py b/spkrepo/utils.py index 749569a..493a241 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -153,22 +153,16 @@ def __init__(self, stream): except UnicodeDecodeError: raise SPKParseError('Wrong conf/PKG_CONX encoding') self.conf_conflicts = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) - if self.conf_dependencies is None and self.conf_conflicts is None: - raise SPKParseError('Empty conf folder') if 'conf/privilege' in names: - c = ConfigParser() try: - c.read_string(spk.extractfile('conf/privilege').read().decode('utf-8')) + self.conf_privilege = spk.extractfile('conf/privilege').read().decode('utf-8').strip() except UnicodeDecodeError: raise SPKParseError('Wrong conf/privilege encoding') - self.conf_privilege = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) if 'conf/resource' in names: - c = ConfigParser() try: - c.read_string(spk.extractfile('conf/resource').read().decode('utf-8')) + self.conf_resource = spk.extractfile('conf/resource').read().decode('utf-8').strip() except UnicodeDecodeError: raise SPKParseError('Wrong conf/resource encoding') - self.conf_resource = json.dumps({s: {k: v for k, v in c.items(s)} for s in c.sections()}) if self.conf_dependencies is None and self.conf_conflicts is None and \ self.conf_privilege is None and self.conf_resource is None: raise SPKParseError('Empty conf folder') From 319f3a4ef3b0e1e7c3d5908ed0e837758da2f50c Mon Sep 17 00:00:00 2001 From: Dr-Bean <github@beanpoint.com> Date: Wed, 21 Dec 2016 12:52:55 +0100 Subject: [PATCH 7/7] Update test_generic and test_missing_conf_folder --- spkrepo/tests/test_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spkrepo/tests/test_utils.py b/spkrepo/tests/test_utils.py index 8561ecd..f424e47 100644 --- a/spkrepo/tests/test_utils.py +++ b/spkrepo/tests/test_utils.py @@ -41,8 +41,9 @@ def test_generic(self): self.assertEqual(build.version.maintainer_url, spk.info['maintainer_url']) self.assertEqual(build.version.package.name, spk.info['package']) self.assertEqual(build.version.report_url, spk.info['report_url']) - self.assertEqual(build.version.conf_dependencies is not None or build.version.conf_conflicts is not None or build.version.conf_privilege is not None, - spk.info['support_conf_folder']) + self.assertEqual(build.version.conf_dependencies is not None or build.version.conf_conflicts is not None or \ + build.version.conf_privilege is not None or build.version.conf_resource is not None, + spk.info['support_conf_folder']) self.assertEqual(build.version.version_string, spk.info['version']) # icons @@ -185,7 +186,7 @@ def test_invalid_spk(self): self.assertEqual('Invalid SPK', str(cm.exception)) def test_missing_conf_folder(self): - build = BuildFactory.build(version__conf_dependencies=None, version__conf_conflicts=None, version__conf_privilege=None) + build = BuildFactory.build(version__conf_dependencies=None, version__conf_conflicts=None, version__conf_privilege=None, version__conf_resource=None) info = create_info(build) info['support_conf_folder'] = 'yes' with create_spk(build, info=info, with_conf=False) as f: