From 869827b23778c4fbb86f2051ad504a1cb4a61d91 Mon Sep 17 00:00:00 2001 From: "ayush.goyal" Date: Wed, 17 Feb 2021 21:57:32 +0530 Subject: [PATCH 1/5] [Django 3.0] Adding compatible versions of other libraries according to Django==3.0.12 --- desktop/core/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/desktop/core/requirements.txt b/desktop/core/requirements.txt index c4bcc599c66..01b86baa16f 100644 --- a/desktop/core/requirements.txt +++ b/desktop/core/requirements.txt @@ -4,12 +4,12 @@ Babel==2.5.1 boto==2.46.1 celery[redis]==4.4.5 # For Python 3.8 cffi==1.13.2 -channels==2.1.6 -channels-redis==2.3.2 +channels==3.0.3 +channels-redis==3.2.0 configobj==5.0.6 cryptography==3.3.2 django-auth-ldap==1.3.0 -Django==2.2.17 +Django==3.0.12 django-axes==5.13.0 django_babel==0.6.2 django-celery-beat==1.4.0 From 718f9614facba3675cabaf7ab563bd7624d16f93 Mon Sep 17 00:00:00 2001 From: "ayush.goyal" Date: Thu, 18 Feb 2021 18:06:58 +0530 Subject: [PATCH 2/5] Adding forked django-babel because the latest one does not support the Django3 --- desktop/core/requirements.txt | 2 +- desktop/core/src/desktop/management/commands/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/desktop/core/requirements.txt b/desktop/core/requirements.txt index 01b86baa16f..74e589c796b 100644 --- a/desktop/core/requirements.txt +++ b/desktop/core/requirements.txt @@ -11,7 +11,6 @@ cryptography==3.3.2 django-auth-ldap==1.3.0 Django==3.0.12 django-axes==5.13.0 -django_babel==0.6.2 django-celery-beat==1.4.0 django_celery_results==1.0.4 django-cors-headers==3.7.0 @@ -63,3 +62,4 @@ sqlparse==0.4.1 tablib==0.13.0 thrift==0.13.0 thrift-sasl==0.4.2 +git+git://github.com/agl29/django-babel.git diff --git a/desktop/core/src/desktop/management/commands/test.py b/desktop/core/src/desktop/management/commands/test.py index 83591a782f3..13d21a42a26 100644 --- a/desktop/core/src/desktop/management/commands/test.py +++ b/desktop/core/src/desktop/management/commands/test.py @@ -28,7 +28,7 @@ from django_nose import runner #import south.management.commands -from django.utils import six +import six from django.utils.translation import deactivate import sys import textwrap From e989639ed0f291d681f03b462f6be17ddf0664d5 Mon Sep 17 00:00:00 2001 From: "ayush.goyal" Date: Thu, 18 Feb 2021 18:35:29 +0530 Subject: [PATCH 3/5] fixing pylint issues --- .../src/desktop/management/commands/test.py | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/desktop/core/src/desktop/management/commands/test.py b/desktop/core/src/desktop/management/commands/test.py index 13d21a42a26..22f6d59f738 100644 --- a/desktop/core/src/desktop/management/commands/test.py +++ b/desktop/core/src/desktop/management/commands/test.py @@ -38,57 +38,57 @@ from desktop.lib import django_mako if six.PY3: - from types import SimpleNamespace + from types import SimpleNamespace else: - class SimpleNamespace(object): - pass + class SimpleNamespace(object): + pass class _TestState(object): - pass + pass def setup_test_environment(debug=None): - """ - Perform global pre-test setup, such as installing the instrumented template - renderer and setting the email backend to the locmem email backend. - """ - if hasattr(_TestState, 'saved_data'): - # Executing this function twice would overwrite the saved values. - raise RuntimeError( - "setup_test_environment() was already called and can't be called " - "again without first calling teardown_test_environment()." - ) + """ + Perform global pre-test setup, such as installing the instrumented template + renderer and setting the email backend to the locmem email backend. + """ + if hasattr(_TestState, 'saved_data'): + # Executing this function twice would overwrite the saved values. + raise RuntimeError( + "setup_test_environment() was already called and can't be called " + "again without first calling teardown_test_environment()." + ) - if debug is None: - debug = settings.DEBUG + if debug is None: + debug = settings.DEBUG - saved_data = SimpleNamespace() - _TestState.saved_data = saved_data + saved_data = SimpleNamespace() + _TestState.saved_data = saved_data - saved_data.allowed_hosts = settings.ALLOWED_HOSTS - # Add the default host of the test client. - settings.ALLOWED_HOSTS = list(settings.ALLOWED_HOSTS) + ['testserver'] + saved_data.allowed_hosts = settings.ALLOWED_HOSTS + # Add the default host of the test client. + settings.ALLOWED_HOSTS = list(settings.ALLOWED_HOSTS) + ['testserver'] - saved_data.debug = settings.DEBUG - settings.DEBUG = debug + saved_data.debug = settings.DEBUG + settings.DEBUG = debug - django_mako.render_to_string = django_mako.render_to_string_test + django_mako.render_to_string = django_mako.render_to_string_test - deactivate() + deactivate() def teardown_test_environment(): - """ - Perform any global post-test teardown, such as restoring the original - template renderer and restoring the email sending functions. - """ - saved_data = _TestState.saved_data + """ + Perform any global post-test teardown, such as restoring the original + template renderer and restoring the email sending functions. + """ + saved_data = _TestState.saved_data - settings.ALLOWED_HOSTS = saved_data.allowed_hosts - settings.DEBUG = saved_data.debug - django_mako.render_to_string = django_mako.render_to_string_normal + settings.ALLOWED_HOSTS = saved_data.allowed_hosts + settings.DEBUG = saved_data.debug + django_mako.render_to_string = django_mako.render_to_string_normal - del _TestState.saved_data + del _TestState.saved_data class Command(BaseCommand): @@ -103,8 +103,8 @@ class Command(BaseCommand): specific Explicitly run specific tests using nose. For example, to run all the filebrower tests or to run a specific test function, use - test specific filebrowser - test specific useradmin.tests:test_user_admin + test specific filebrowser + test specific useradmin.tests:test_user_admin All additional arguments are passed directly to nose. list_modules List test modules for all desktop applications and libraries @@ -141,7 +141,7 @@ def run_from_argv(self, argv): sys.exit(1) nose_args = None - all_apps = [ app.module.__name__ for app in appmanager.DESKTOP_MODULES ] + all_apps = [app.module.__name__ for app in appmanager.DESKTOP_MODULES] if args[0] == "all": nose_args = args + all_apps From 46927fbd31d9d4ed30889c412de5c4d57747a38f Mon Sep 17 00:00:00 2001 From: "ayush.goyal" Date: Fri, 19 Feb 2021 17:04:02 +0530 Subject: [PATCH 4/5] Changes made to pass the unit tests --- apps/beeswax/src/beeswax/tests.py | 5 ++++- apps/jobbrowser/src/jobbrowser/tests.py | 6 +++++- apps/oozie/src/oozie/views/editor.py | 2 +- apps/useradmin/src/useradmin/tests.py | 10 ++++++++-- desktop/core/requirements.txt | 2 +- desktop/core/src/desktop/ldaptestcmd_tests.py | 2 +- 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/apps/beeswax/src/beeswax/tests.py b/apps/beeswax/src/beeswax/tests.py index f65a946be00..fd7c972fd70 100644 --- a/apps/beeswax/src/beeswax/tests.py +++ b/apps/beeswax/src/beeswax/tests.py @@ -2910,7 +2910,10 @@ def test_save_design(self): def test_get_history_xss(self): sql = 'SELECT count(sample_07.salary) FROM sample_07;">' - sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert('Hue has an xss');"></iFraME>' + if sys.version_info[0] < 3: + sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert('Hue has an xss');"></iFraME>' + else: + sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert('Hue has an xss');"></iFraME>' response = _make_query(self.client, sql, submission_type='Save', name='My Name 1', desc='My Description') content = json.loads(response.content) diff --git a/apps/jobbrowser/src/jobbrowser/tests.py b/apps/jobbrowser/src/jobbrowser/tests.py index 4027a2476c9..ade71df82da 100644 --- a/apps/jobbrowser/src/jobbrowser/tests.py +++ b/apps/jobbrowser/src/jobbrowser/tests.py @@ -21,6 +21,7 @@ import json import logging import re +import sys import time import unittest @@ -490,7 +491,10 @@ def test_acls_job(self): assert_false(can_modify_job('test3', response.context[0]['job'])) response2 = self.c3.get('/jobbrowser/jobs/job_1356251510842_0054') - assert_true(b'don't have permission to access job' in response2.content, response2.content) + if sys.version_info[0] < 3: + assert_true(b'don't have permission to access job' in response2.content, response2.content) + else: + assert_true(b'don't have permission to access job' in response2.content, response2.content) def test_kill_job(self): job_id = 'application_1356251510842_0054' diff --git a/apps/oozie/src/oozie/views/editor.py b/apps/oozie/src/oozie/views/editor.py index b0b8596c9d8..3fc8cd8c5f8 100644 --- a/apps/oozie/src/oozie/views/editor.py +++ b/apps/oozie/src/oozie/views/editor.py @@ -398,7 +398,7 @@ def schedule_workflow(request, workflow): @check_job_access_permission() -def create_coordinator(request, workflow=None): +def create_coordinator(request, workflow=None, *args, **kwargs): if workflow is not None: coordinator = Coordinator(owner=request.user, schema_version="uri:oozie:coordinator:0.2", coordinatorworkflow=workflow) else: diff --git a/apps/useradmin/src/useradmin/tests.py b/apps/useradmin/src/useradmin/tests.py index b32e2bff60d..89761ada1f8 100644 --- a/apps/useradmin/src/useradmin/tests.py +++ b/apps/useradmin/src/useradmin/tests.py @@ -1111,7 +1111,10 @@ def test_edit_user_xss(self): language="en-us>" ) ) - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + if sys.version_info[0] < 3: + assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + else: + assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) # Hue 4 Admin response = edit_user.post('/useradmin/users/edit/admin', dict( username="admin", @@ -1133,7 +1136,10 @@ def test_edit_user_xss(self): language="en-us>" ) ) - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + if sys.version_info[0] < 3: + assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + else: + assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) # Hue 4, User with access to useradmin app response = edit_user.post('/useradmin/users/edit/edit_user', dict( username="edit_user", diff --git a/desktop/core/requirements.txt b/desktop/core/requirements.txt index 74e589c796b..0c01c6511d8 100644 --- a/desktop/core/requirements.txt +++ b/desktop/core/requirements.txt @@ -8,7 +8,7 @@ channels==3.0.3 channels-redis==3.2.0 configobj==5.0.6 cryptography==3.3.2 -django-auth-ldap==1.3.0 +django-auth-ldap==2.3.0 Django==3.0.12 django-axes==5.13.0 django-celery-beat==1.4.0 diff --git a/desktop/core/src/desktop/ldaptestcmd_tests.py b/desktop/core/src/desktop/ldaptestcmd_tests.py index f8e0711e606..2f24fd3eda8 100644 --- a/desktop/core/src/desktop/ldaptestcmd_tests.py +++ b/desktop/core/src/desktop/ldaptestcmd_tests.py @@ -21,7 +21,7 @@ from django.core import management from django.core.management import get_commands from django.test import SimpleTestCase -from django.utils.six import StringIO +from six import StringIO class CmdTests(SimpleTestCase): def checkcmd(self): From a838251f667d8ae95abf5d4c413fef49a10e4c23 Mon Sep 17 00:00:00 2001 From: "ayush.goyal" Date: Sat, 20 Feb 2021 00:13:50 +0530 Subject: [PATCH 5/5] fixing pylint issues --- apps/beeswax/src/beeswax/tests.py | 201 ++++++++++++++++-------- apps/useradmin/src/useradmin/tests.py | 212 ++++++++++++++++++-------- 2 files changed, 282 insertions(+), 131 deletions(-) diff --git a/apps/beeswax/src/beeswax/tests.py b/apps/beeswax/src/beeswax/tests.py index fd7c972fd70..4de301badac 100644 --- a/apps/beeswax/src/beeswax/tests.py +++ b/apps/beeswax/src/beeswax/tests.py @@ -117,7 +117,7 @@ def _make_query(client, query, submission_type="Execute", return res def random_generator(size=8, chars=string.ascii_uppercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) + return ''.join(random.choice(chars) for _ in range(size)) def get_csv(client, result_response): """Get the csv for a query result""" @@ -219,9 +219,12 @@ def test_query_with_resource(self): def test_query_with_setting(self): table_name = 'test_query_with_setting' - response = _make_query(self.client, "CREATE TABLE `%(db)s`.`%(table_name)s` AS SELECT foo+1 FROM test WHERE foo=4" % {'db': self.db_name, 'table_name': table_name}, - settings=[("mapred.job.name", "test_query_with_setting"), - ("hive.exec.compress.output", "true")], local=False, database=self.db_name) # Run on MR, because that's how we check it worked. + response = _make_query( + self.client, + "CREATE TABLE `%(db)s`.`%(table_name)s` AS SELECT foo+1 FROM test WHERE foo=4" % {'db': self.db_name, 'table_name': table_name}, + settings=[("mapred.job.name", "test_query_with_setting"), ("hive.exec.compress.output", "true")], + local=False, database=self.db_name + ) # Run on MR, because that's how we check it worked. response = wait_for_query_to_finish(self.client, response, max=180.0) # Check that we actually got a compressed output table = self.db.get_table(database=self.db_name, table_name=table_name) @@ -233,12 +236,12 @@ def test_query_with_setting(self): raise SkipTest # And check that the name is right... - assert_true("test_query_with_setting" in [ x.profile.name for x in self.cluster.jt.all_jobs().jobs ]) + assert_true("test_query_with_setting" in [x.profile.name for x in self.cluster.jt.all_jobs().jobs]) # While we're at it, check that we're running jobs as the correct user on MR. assert_equal("test", - [ x.profile for x in self.cluster.jt.all_jobs().jobs - if x.profile.name == "test_query_with_setting" ][0].user) + [x.profile for x in self.cluster.jt.all_jobs().jobs + if x.profile.name == "test_query_with_setting"][0].user) def test_lazy_query_status_update(self): @@ -418,10 +421,11 @@ def test_query_with_udf(self): execution_engines = get_available_execution_engines() for engine in execution_engines: - response = _make_query(self.client, "SELECT my_sqrt(foo), my_float(foo) FROM test where foo=4 GROUP BY foo", # Force MR job with GROUP BY - udfs=[('my_sqrt', 'org.apache.hadoop.hive.ql.udf.UDFSqrt'), - ('my_float', 'org.apache.hadoop.hive.ql.udf.UDFToFloat')], - local=False, database=self.db_name, settings=[('hive.execution.engine', engine)]) + response = _make_query( + self.client, "SELECT my_sqrt(foo), my_float(foo) FROM test where foo=4 GROUP BY foo", # Force MR job with GROUP BY + udfs=[('my_sqrt', 'org.apache.hadoop.hive.ql.udf.UDFSqrt'), ('my_float', 'org.apache.hadoop.hive.ql.udf.UDFToFloat')], + local=False, database=self.db_name, settings=[('hive.execution.engine', engine)] + ) response = wait_for_query_to_finish(self.client, response, max=60.0) content = fetch_query_result_data(self.client, response) @@ -559,7 +563,9 @@ def test_parameterization(self): ], content['parameters'], content) # Now fill it out - response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", params=[('x', '1'), ('y', '2')], database=self.db_name) + response = _make_query( + self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", params=[('x', '1'), ('y', '2')], database=self.db_name + ) content = json.loads(response.content) assert_true('watch_url' in content, content) query_history = QueryHistory.get(content['id']) @@ -568,7 +574,10 @@ def test_parameterization(self): assert_equal("SELECT foo FROM test WHERE foo='1' and bar='2'", query_history.query) # Check that error handling is reasonable - response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", params=[('x', "'_this_is_not SQL "), ('y', '2')], database=self.db_name) + response = _make_query( + self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", + params=[('x', "'_this_is_not SQL "), ('y', '2')], database=self.db_name + ) content = json.loads(response.content) assert_true("FAILED: ParseException" in content.get('message'), content) @@ -594,7 +603,10 @@ def test_explain_query(self): response = _make_query(c, CREATE_TABLE, database=self.db_name) wait_for_query_to_finish(c, response) - response = _make_query(c, "SELECT SUM(foo) FROM `%(db)s`.`test_explain`" % {'db': self.db_name}, settings=[('hive.explain.user', 'false')], submission_type="Explain") # Need to prefix database in Explain + response = _make_query( + c, "SELECT SUM(foo) FROM `%(db)s`.`test_explain`" % {'db': self.db_name}, + settings=[('hive.explain.user', 'false')], submission_type="Explain" + ) # Need to prefix database in Explain explanation = json.loads(response.content)['explanation'] assert_true('STAGE DEPENDENCIES:' in explanation, explanation) assert_true('STAGE PLANS:' in explanation, explanation) @@ -805,7 +817,7 @@ def test_parallel_queries(self): raise SkipTest # sqlite does not support concurrent transaction PARALLEL_TASKS = 2 - responses = [ None ] * PARALLEL_TASKS + responses = [None] * PARALLEL_TASKS threads = [] # Protects responses lock = threading.Lock() @@ -825,8 +837,8 @@ def test_parallel_queries(self): for i in range(PARALLEL_TASKS): csv = get_csv(self.client, responses[i]) # We get 3 rows: Column header, and 2 rows of results in double quotes - answer = [ int(data.strip('"')) for data in csv.split()[1:] ] - assert_equal( [ i + 1, i + 2 ], answer) + answer = [int(data.strip('"')) for data in csv.split()[1:]] + assert_equal([i + 1, i + 2], answer) def test_data_export_limit_clause(self): @@ -1488,7 +1500,7 @@ def test_create_table_import(self): ] def write_file(filename, raw_fields, delim, do_gzip=False): - lines = [ delim.join(row) for row in raw_fields ] + lines = [delim.join(row) for row in raw_fields] data = '\n'.join(lines) if do_gzip: sio = string_io() @@ -1567,7 +1579,7 @@ def write_file(filename, raw_fields, delim, do_gzip=False): ['a', 'b', 'c'], ['a,a', 'b,b', 'c,c'], ['a,"a', 'b,"b', 'c,"c'], - ] ) + ]) # Test column definition resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, { @@ -1637,7 +1649,7 @@ def __init__(self, content): resp = self.client.get(on_success_url_show_table) cols = resp.context[0]['table'].cols assert_equal(len(cols), 3) - assert_equal([ col.name for col in cols ], [ 'col_a', 'col_b', 'col_c' ]) + assert_equal([col.name for col in cols], ['col_a', 'col_b', 'col_c']) resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'test_create_import'})) rows = json.loads(resp.content)['rows'] flat_rows = sum(rows, []) @@ -1961,7 +1973,9 @@ def test_analyze_table_and_read_statistics(self): response = wait_for_query_to_finish(self.client, response, max=120.0) assert_true(response, response) - response = self.client.post(reverse("beeswax:analyze_table", kwargs={'database': self.db_name, 'table': 'test', 'columns': True}), follow=True) + response = self.client.post( + reverse("beeswax:analyze_table", kwargs={'database': self.db_name, 'table': 'test', 'columns': True}), follow=True + ) response = wait_for_query_to_finish(self.client, response, max=120.0) assert_true(response, response) @@ -2004,7 +2018,9 @@ def test_get_top_terms(self): assert_equal([[255, 1], [254, 1], [253, 1], [252, 1]], terms[:4]) - resp = self.client.get(reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'})) + resp = self.client.get( + reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'}) + ) content = json.loads(resp.content) assert_true('terms' in content, 'Failed to get terms: %s' % (content,)) @@ -2012,7 +2028,9 @@ def test_get_top_terms(self): assert_equal([[109, 1], [108, 1], [107, 1], [106, 1]], terms[:4]) - resp = self.client.get(reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'}) + '?limit=2') + resp = self.client.get( + reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'}) + '?limit=2' + ) content = json.loads(resp.content) assert_true('terms' in content, 'Failed to get terms: %s' % (content,)) @@ -2042,7 +2060,9 @@ def test_beeswax_api_autocomplete(self): assert_equal({'comment': '', 'type': 'array>', 'name': 'foo'}, extended_columns[0]) # Autocomplete nested fields for a given column - resp = self.client.get(reverse("beeswax:api_autocomplete_column", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo'})) + resp = self.client.get( + reverse("beeswax:api_autocomplete_column", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo'}) + ) json_resp = json.loads(resp.content) assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error')) @@ -2051,7 +2071,9 @@ def test_beeswax_api_autocomplete(self): assert_equal("struct", json_resp["item"]["type"]) # Autocomplete nested fields for a given nested type - resp = self.client.get(reverse("beeswax:api_autocomplete_nested", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo', 'nested': 'item'})) + resp = self.client.get(reverse( + "beeswax:api_autocomplete_nested", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo', 'nested': 'item'} + )) json_resp = json.loads(resp.content) assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error')) @@ -2313,7 +2335,10 @@ def get(self): return tmpdir thrift_uris = 'thrift://%s:9999' % hostname - xml = hive_site_xml(is_local=False, use_sasl=False, thrift_uris=thrift_uris, kerberos_principal='test/_HOST@TEST.COM', hs2_kerberos_principal='test/_HOST@TEST.COM') + xml = hive_site_xml( + is_local=False, use_sasl=False, thrift_uris=thrift_uris, kerberos_principal='test/_HOST@TEST.COM', + hs2_kerberos_principal='test/_HOST@TEST.COM' + ) open_file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml) beeswax.hive_site.reset() @@ -2375,7 +2400,8 @@ def test_search_log_line(): """ assert_true(search_log_line('FAILED: Parse Error', logs)) - logs = "12/08/22 20:50:14 ERROR ql.Driver: FAILED: Parse Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant'" + logs = "12/08/22 20:50:14 ERROR ql.Driver: FAILED: Parse Error: line 1:31 cannot recognize input "\ + "near '''' '_this_is_not' 'SQL' in constant'" assert_true(search_log_line('FAILED: Parse Error', logs)) logs = """ @@ -2422,14 +2448,20 @@ def test_search_log_line(): USE functional; ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=1); ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2);""" - assert_equal(['CREATE DATABASE IF NOT EXISTS functional', - 'DROP TABLE IF EXISTS functional.alltypes', - "CREATE EXTERNAL TABLE IF NOT EXISTS functional.alltypes (\nid int COMMENT 'Add a comment',\nbool_col boolean,\ntinyint_col tinyint,\nsmallint_col smallint,\nint_col int,\nbigint_col bigint,\nfloat_col float,\ndouble_col double,\ndate_string_col string,\nstring_col string,\ntimestamp_col timestamp)\nPARTITIONED BY (year int, month int)\nROW FORMAT delimited fields terminated by ',' escaped by '\\\\'\nSTORED AS TEXTFILE\nLOCATION '/user/admin/alltypes/alltypes'", - 'USE functional', - 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=1)', - 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2)' - ], - hql_query(query).statements, hql_query(query).statements) + assert_equal( + [ + 'CREATE DATABASE IF NOT EXISTS functional', + 'DROP TABLE IF EXISTS functional.alltypes', + "CREATE EXTERNAL TABLE IF NOT EXISTS functional.alltypes (\nid int COMMENT 'Add a comment',\nbool_col boolean,\ntinyint_col "\ + "tinyint,\nsmallint_col smallint,\nint_col int,\nbigint_col bigint,\nfloat_col float,\ndouble_col double,\ndate_string_col "\ + "string,\nstring_col string,\ntimestamp_col timestamp)\nPARTITIONED BY (year int, month int)\nROW FORMAT delimited fields "\ + "terminated by ',' escaped by '\\\\'\nSTORED AS TEXTFILE\nLOCATION '/user/admin/alltypes/alltypes'", + 'USE functional', + 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=1)', + 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2)' + ], + hql_query(query).statements, hql_query(query).statements + ) class MockHiveServerTable(HiveServerTable): @@ -2465,7 +2497,9 @@ def __init__(self, describe=None): {'comment': None, 'col_name': '# Storage Information', 'data_type': None}, {'comment': None, 'col_name': 'SerDe Library: ', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'}, {'comment': None, 'col_name': 'InputFormat: ', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'}, - {'comment': None, 'col_name': 'OutputFormat: ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'}, + { + 'comment': None, 'col_name': 'OutputFormat: ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' + }, {'comment': None, 'col_name': 'Compressed: ', 'data_type': 'No '}, {'comment': None, 'col_name': 'Num Buckets: ', 'data_type': '-1 '}, {'comment': None, 'col_name': 'Bucket Columns: ', 'data_type': '[] '}, @@ -2693,9 +2727,14 @@ def test_column_format_values_nulls(self): data = [1] * 8 * 8 nulls = '\x01\x23\x45\x67\x89\xab\xcd\xef' - assert_equal([None, 1, 1, 1, 1, 1, 1, 1, None, None, 1, 1, 1, None, 1, 1, None, 1, None, 1, 1, 1, None, 1, None, None, None, 1, 1, None, None, 1, None, 1, 1, - None, 1, 1, 1, None, None, None, 1, None, 1, None, 1, None, None, 1, None, None, 1, 1, None, None, None, None, None, None, 1, None, None, None], - HiveServerTColumnValue2.set_nulls(data, nulls)) + assert_equal( + [ + None, 1, 1, 1, 1, 1, 1, 1, None, None, 1, 1, 1, None, 1, 1, None, 1, None, 1, 1, 1, None, 1, None, None, None, 1, 1, None, None, 1, + None, 1, 1, None, 1, 1, 1, None, None, None, 1, None, 1, None, 1, None, None, 1, None, None, 1, 1, None, None, None, None, None, + None, 1, None, None, None + ], + HiveServerTColumnValue2.set_nulls(data, nulls) + ) def test_column_detect_if_values_nulls(self): @@ -2840,7 +2879,10 @@ def test_save_design(self): design_obj = hql_query('SELECT') # Save his own query - saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc') + saved_design = _save_design( + user=self.user, design=design, type_=HQL, design_obj=design_obj, + explicit_save=True, name='test_save_design', desc='test_save_design desc' + ) assert_equal('test_save_design', saved_design.name) assert_equal('test_save_design desc', saved_design.desc) assert_equal('test_save_design', saved_design.doc.get().name) @@ -2848,7 +2890,10 @@ def test_save_design(self): assert_false(saved_design.doc.get().is_historic()) # Execute it as auto - saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=False, name='test_save_design', desc='test_save_design desc') + saved_design = _save_design( + user=self.user, design=design, type_=HQL, design_obj=design_obj, + explicit_save=False, name='test_save_design', desc='test_save_design desc' + ) assert_equal('test_save_design (new)', saved_design.name) assert_equal('test_save_design desc', saved_design.desc) assert_equal('test_save_design (new)', saved_design.doc.get().name) @@ -2857,7 +2902,10 @@ def test_save_design(self): # not_me user can't modify it try: - _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc') + _save_design( + user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, + explicit_save=True, name='test_save_design', desc='test_save_design desc' + ) assert_true(False, 'not_me is not allowed') except PopupException: pass @@ -2865,13 +2913,19 @@ def test_save_design(self): saved_design.doc.get().share_to_default() try: - _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc') + _save_design( + user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, + explicit_save=True, name='test_save_design', desc='test_save_design desc' + ) assert_true(False, 'not_me is not allowed') except PopupException: pass # not_me can execute it as auto - saved_design = _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=False, name='test_save_design', desc='test_save_design desc') + saved_design = _save_design( + user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, + explicit_save=False, name='test_save_design', desc='test_save_design desc' + ) assert_equal('test_save_design (new)', saved_design.name) assert_equal('test_save_design desc', saved_design.desc) assert_equal('test_save_design (new)', saved_design.doc.get().name) @@ -2881,7 +2935,10 @@ def test_save_design(self): # not_me can save as a new design design = SavedQuery(owner=self.user_not_me, type=HQL) - saved_design = _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc') + saved_design = _save_design( + user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, + explicit_save=True, name='test_save_design', desc='test_save_design desc' + ) assert_equal('test_save_design', saved_design.name) assert_equal('test_save_design desc', saved_design.desc) assert_equal('test_save_design', saved_design.doc.get().name) @@ -2898,12 +2955,16 @@ def test_save_design(self): design_obj = hql_query('SELECT') # Save query - saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, - explicit_save=True, name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc') + saved_design = _save_design( + user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, + name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc' + ) len_after = len(saved_design.name) assert_equal(len_after, 64) - saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, - explicit_save=False, name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc') + saved_design = _save_design( + user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=False, + name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc' + ) # Above design name is already 64 characters, so saved_design name shouldn't exceed the limit len_after = len(saved_design.name) assert_equal(len_after, 64) @@ -2911,9 +2972,11 @@ def test_save_design(self): def test_get_history_xss(self): sql = 'SELECT count(sample_07.salary) FROM sample_07;">' if sys.version_info[0] < 3: - sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert('Hue has an xss');"></iFraME>' + sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>'\ + b'src="javascript:alert('Hue has an xss');"></iFraME>' else: - sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert('Hue has an xss');"></iFraME>' + sql_escaped = b'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>'\ + b'src="javascript:alert('Hue has an xss');"></iFraME>' response = _make_query(self.client, sql, submission_type='Save', name='My Name 1', desc='My Description') content = json.loads(response.content) @@ -3486,13 +3549,13 @@ def test_ssl_validate(): def test_to_matching_wildcard(): - match_fn = dbms.HiveServer2Dbms.to_matching_wildcard + match_fn = dbms.HiveServer2Dbms.to_matching_wildcard - assert_equal(match_fn(None), '*') - assert_equal(match_fn(''), '*') - assert_equal(match_fn('*'), '*') - assert_equal(match_fn('test'), '*test*') - assert_equal(match_fn('test*'), '*test*') + assert_equal(match_fn(None), '*') + assert_equal(match_fn(''), '*') + assert_equal(match_fn('*'), '*') + assert_equal(match_fn('test'), '*test*') + assert_equal(match_fn('test*'), '*test*') def test_apply_natural_sort(): @@ -3528,14 +3591,19 @@ def test_hiveserver2_jdbc_url(): beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_TRUSTSTORE_PATH] = '/path/to/truststore.jks' beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_TRUSTSTORE_PASSWORD] = 'password' url = hiveserver2_jdbc_url() - assert_equal(url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default;ssl=true;sslTrustStore=/path/to/truststore.jks;trustStorePassword=password') + assert_equal( + url, + 'jdbc:hive2://server-with-ssl-enabled.com:10000/default;ssl=true;sslTrustStore=/path/to/truststore.jks;trustStorePassword=password' + ) beeswax.hive_site.reset() beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_USE_SSL] = 'TRUE' hadoop.ssl_client_site.reset() hadoop.ssl_client_site.get_conf()[ssl_client_site._CNF_TRUSTORE_LOCATION] = '/etc/ssl-conf/CA_STANDARD/truststore.jks' url = hiveserver2_jdbc_url() # Pick-up trustore from ssl-client.xml - assert_equal(url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default;ssl=true;sslTrustStore=/etc/ssl-conf/CA_STANDARD/truststore.jks') + assert_equal( + url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default;ssl=true;sslTrustStore=/etc/ssl-conf/CA_STANDARD/truststore.jks' + ) beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_USE_SSL] = 'FALSE' url = hiveserver2_jdbc_url() @@ -3548,7 +3616,7 @@ def test_hiveserver2_jdbc_url(): beeswax.hive_site.reset() hadoop.ssl_client_site.reset() for reset in resets: - reset() + reset() def test_sasl_auth_in_large_download(): db = None @@ -3569,14 +3637,15 @@ def test_sasl_auth_in_large_download(): # Create a big table table_info = {'db': 'default', 'table_name': 'dummy_'+random_generator().lower()} drop_sql = "DROP TABLE IF EXISTS %(db)s.%(table_name)s" % table_info - create_sql = "CREATE TABLE IF NOT EXISTS %(db)s.%(table_name)s (w0 CHAR(8),w1 CHAR(8),w2 CHAR(8),w3 CHAR(8),w4 CHAR(8),w5 CHAR(8),w6 CHAR(8),w7 CHAR(8),w8 CHAR(8),w9 CHAR(8))" % table_info + create_sql = "CREATE TABLE IF NOT EXISTS %(db)s.%(table_name)s (w0 CHAR(8),w1 CHAR(8),w2 CHAR(8),w3 CHAR(8),w4 "\ + "CHAR(8),w5 CHAR(8),w6 CHAR(8),w7 CHAR(8),w8 CHAR(8),w9 CHAR(8))" % table_info hql = string_io() hql.write("INSERT INTO %(db)s.%(table_name)s VALUES " % (table_info)) for i in range(max_rows-1): w = random_generator(size=7) - hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')," % (w,w,w,w,w,w,w,w,w,w)) + hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')," % (w, w, w, w, w, w, w, w, w, w)) w = random_generator(size=7) - hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')" % (w,w,w,w,w,w,w,w,w,w)) + hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')" % (w, w, w, w, w, w, w, w, w, w)) try: db = dbms.get(user, get_query_server_config()) @@ -3608,7 +3677,7 @@ def test_sasl_auth_in_large_download(): if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message: failed = True except: - failed = True + failed = True # Fetch large data set is successful because SASL_MAX_BUFFER > RESULT_DATA assert_false(failed) @@ -3622,7 +3691,7 @@ def test_sasl_auth_in_large_download(): if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message: failed = True except: - failed = True + failed = True # Fetch large data set fails because SASL_MAX_BUFFER < RESULT_DATA In your log file you will see following log lines # thrift_util INFO Thrift exception; retrying: Error in sasl_decode (-1) SASL(-1): generic failure: Unable to find a callback: 32775 @@ -3640,5 +3709,5 @@ def test_sasl_auth_in_large_download(): if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message: failed = True except: - failed = True + failed = True assert_false(failed) diff --git a/apps/useradmin/src/useradmin/tests.py b/apps/useradmin/src/useradmin/tests.py index 89761ada1f8..f7e1abfac8f 100644 --- a/apps/useradmin/src/useradmin/tests.py +++ b/apps/useradmin/src/useradmin/tests.py @@ -109,13 +109,14 @@ def find_users(self, username_pattern, search_attr=None, user_name_attr=None, fi if find_by_dn: data = [attrs for attrs in list(self._instance.users.values()) if attrs['dn'] == username_pattern] else: - username_pattern = "^%s$" % username_pattern.replace('.','\\.').replace('*', '.*') + username_pattern = "^%s$" % username_pattern.replace('.', '\\.').replace('*', '.*') username_fsm = re.compile(username_pattern, flags=re.I) usernames = [username for username in list(self._instance.users.keys()) if username_fsm.match(username)] data = [self._instance.users.get(username) for username in usernames] return data - def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, group_member_attr=None, group_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE): + def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, + group_member_attr=None, group_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE): """ Return all groups in the system with parents and children """ if find_by_dn: data = [attrs for attrs in list(self._instance.groups.values()) if attrs['dn'] == groupname_pattern] @@ -124,7 +125,7 @@ def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, sub_data = [attrs for attrs in list(self._instance.groups.values()) if attrs['dn'].endswith(data[0]['dn'])] data.extend(sub_data) else: - groupname_pattern = "^%s$" % groupname_pattern.replace('.','\\.').replace('*', '.*') + groupname_pattern = "^%s$" % groupname_pattern.replace('.', '\\.').replace('*', '.*') groupnames = [username for username in list(self._instance.groups.keys()) if re.match(groupname_pattern, username)] data = [self._instance.groups.get(groupname) for groupname in groupnames] return data @@ -179,63 +180,108 @@ class Data(object): def __init__(self): long_username = create_long_username() self.users = { - 'moe': {'dn': 'uid=moe,ou=People,dc=example,dc=com', 'username':'moe', 'first':'Moe', 'email':'moe@stooges.com', 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com']}, - 'lårry': {'dn': 'uid=lårry,ou=People,dc=example,dc=com', 'username':'lårry', 'first':'Larry', 'last':'Stooge', 'email':'larry@stooges.com', 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']}, - 'curly': {'dn': 'uid=curly,ou=People,dc=example,dc=com', 'username':'curly', 'first':'Curly', 'last':'Stooge', 'email':'curly@stooges.com', 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']}, - 'Rock': {'dn': 'uid=Rock,ou=People,dc=example,dc=com', 'username':'Rock', 'first':'rock', 'last':'man', 'email':'rockman@stooges.com', 'groups': ['cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']}, - 'nestedguy': {'dn': 'uid=nestedguy,ou=People,dc=example,dc=com', 'username':'nestedguy', 'first':'nested', 'last':'guy', 'email':'nestedguy@stooges.com', 'groups': ['cn=NestedGroup,ou=Groups,dc=example,dc=com']}, - 'otherguy': {'dn': 'uid=otherguy,ou=People,dc=example,dc=com', 'username':'otherguy', 'first':'Other', 'last':'Guy', 'email':'other@guy.com'}, - 'posix_person': {'dn': 'uid=posix_person,ou=People,dc=example,dc=com', 'username': 'posix_person', 'first': 'pos', 'last': 'ix', 'email': 'pos@ix.com'}, - 'posix_person2': {'dn': 'uid=posix_person2,ou=People,dc=example,dc=com', 'username': 'posix_person2', 'first': 'pos', 'last': 'ix', 'email': 'pos@ix.com'}, - 'user with space': {'dn': 'uid=user with space,ou=People,dc=example,dc=com', 'username': 'user with space', 'first': 'user', 'last': 'space', 'email': 'user@space.com'}, - 'spaceless': {'dn': 'uid=user without space,ou=People,dc=example,dc=com', 'username': 'spaceless', 'first': 'user', 'last': 'space', 'email': 'user@space.com'}, - long_username: {'dn': 'uid=' + long_username + ',ou=People,dc=example,dc=com', 'username': long_username, 'first': 'toolong', 'last': 'username', 'email': 'toolong@username.com'}, - 'test_longfirstname': {'dn': 'uid=test_longfirstname,ou=People,dc=example,dc=com', 'username': 'test_longfirstname', 'first': 'test_longfirstname_test_longfirstname', 'last': 'username', 'email': 'toolong@username.com'},} + 'moe': { + 'dn': 'uid=moe,ou=People,dc=example,dc=com', 'username': 'moe', 'first': 'Moe', 'email': 'moe@stooges.com', + 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com'] + }, + 'lårry': { + 'dn': 'uid=lårry,ou=People,dc=example,dc=com', 'username': 'lårry', 'first': 'Larry', 'last': 'Stooge', + 'email': 'larry@stooges.com', + 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com'] + }, + 'curly': { + 'dn': 'uid=curly,ou=People,dc=example,dc=com', 'username': 'curly', 'first': 'Curly', 'last': 'Stooge', + 'email': 'curly@stooges.com', + 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com'] + }, + 'Rock': { + 'dn': 'uid=Rock,ou=People,dc=example,dc=com', 'username': 'Rock', 'first': 'rock', 'last': 'man', 'email': 'rockman@stooges.com', + 'groups': ['cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com'] + }, + 'nestedguy': { + 'dn': 'uid=nestedguy,ou=People,dc=example,dc=com', 'username': 'nestedguy', 'first': 'nested', 'last': 'guy', + 'email': 'nestedguy@stooges.com', 'groups': ['cn=NestedGroup,ou=Groups,dc=example,dc=com'] + }, + 'otherguy': { + 'dn': 'uid=otherguy,ou=People,dc=example,dc=com', 'username': 'otherguy', 'first': 'Other', 'last': 'Guy', + 'email': 'other@guy.com' + }, + 'posix_person': { + 'dn': 'uid=posix_person,ou=People,dc=example,dc=com', 'username': 'posix_person', 'first': 'pos', 'last': 'ix', + 'email': 'pos@ix.com' + }, + 'posix_person2': { + 'dn': 'uid=posix_person2,ou=People,dc=example,dc=com', 'username': 'posix_person2', 'first': 'pos', 'last': 'ix', + 'email': 'pos@ix.com' + }, + 'user with space': { + 'dn': 'uid=user with space,ou=People,dc=example,dc=com', 'username': 'user with space', 'first': 'user', 'last': 'space', + 'email': 'user@space.com' + }, + 'spaceless': { + 'dn': 'uid=user without space,ou=People,dc=example,dc=com', 'username': 'spaceless', 'first': 'user', 'last': 'space', + 'email': 'user@space.com' + }, + long_username: { + 'dn': 'uid=' + long_username + ',ou=People,dc=example,dc=com', 'username': long_username, 'first': 'toolong', 'last': 'username', + 'email': 'toolong@username.com' + }, + 'test_longfirstname': { + 'dn': 'uid=test_longfirstname,ou=People,dc=example,dc=com', 'username': 'test_longfirstname', + 'first': 'test_longfirstname_test_longfirstname', 'last': 'username', 'email': 'toolong@username.com' + }, + } self.groups = { 'TestUsers': { 'dn': 'cn=TestUsers,ou=Groups,dc=example,dc=com', - 'name':'TestUsers', - 'members':['uid=moe,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com','uid=' + long_username + ',ou=People,dc=example,dc=com'], - 'posix_members':[]}, + 'name': 'TestUsers', + 'members': [ + 'uid=moe,ou=People,dc=example,dc=com', 'uid=lårry,ou=People,dc=example,dc=com', + 'uid=curly,ou=People,dc=example,dc=com', 'uid=' + long_username + ',ou=People,dc=example,dc=com' + ], + 'posix_members': []}, 'Test Administrators': { 'dn': 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com', - 'name':'Test Administrators', - 'members':['uid=Rock,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com','uid=' + long_username + ',ou=People,dc=example,dc=com'], - 'posix_members':[]}, + 'name': 'Test Administrators', + 'members': [ + 'uid=Rock,ou=People,dc=example,dc=com', 'uid=lårry,ou=People,dc=example,dc=com', + 'uid=curly,ou=People,dc=example,dc=com', 'uid=' + long_username + ',ou=People,dc=example,dc=com' + ], + 'posix_members': []}, 'OtherGroup': { 'dn': 'cn=OtherGroup,cn=TestUsers,ou=Groups,dc=example,dc=com', - 'name':'OtherGroup', - 'members':[], - 'posix_members':[]}, + 'name': 'OtherGroup', + 'members': [], + 'posix_members': []}, 'NestedGroups': { 'dn': 'cn=NestedGroups,ou=Groups,dc=example,dc=com', - 'name':'NestedGroups', - 'members':['cn=NestedGroup,ou=Groups,dc=example,dc=com'], - 'posix_members':[] + 'name': 'NestedGroups', + 'members': ['cn=NestedGroup,ou=Groups,dc=example,dc=com'], + 'posix_members': [] }, 'NestedGroup': { 'dn': 'cn=NestedGroup,ou=Groups,dc=example,dc=com', - 'name':'NestedGroup', - 'members':['uid=nestedguy,ou=People,dc=example,dc=com'], - 'posix_members':[] + 'name': 'NestedGroup', + 'members': ['uid=nestedguy,ou=People,dc=example,dc=com'], + 'posix_members': [] }, 'NestedPosixGroups': { 'dn': 'cn=NestedPosixGroups,ou=Groups,dc=example,dc=com', - 'name':'NestedPosixGroups', - 'members':['cn=PosixGroup,ou=Groups,dc=example,dc=com'], - 'posix_members':[] + 'name': 'NestedPosixGroups', + 'members': ['cn=PosixGroup,ou=Groups,dc=example,dc=com'], + 'posix_members': [] }, 'PosixGroup': { 'dn': 'cn=PosixGroup,ou=Groups,dc=example,dc=com', - 'name':'PosixGroup', - 'members':[], - 'posix_members':['posix_person','lårry']}, + 'name': 'PosixGroup', + 'members': [], + 'posix_members': ['posix_person', 'lårry']}, 'PosixGroup1': { 'dn': 'cn=PosixGroup1,cn=PosixGroup,ou=Groups,dc=example,dc=com', - 'name':'PosixGroup1', - 'members':[], - 'posix_members':['posix_person2']}, + 'name': 'PosixGroup1', + 'members': [], + 'posix_members': ['posix_person2']}, } def create_long_username(): @@ -399,7 +445,7 @@ def test_group_permissions(self): c.post('/useradmin/groups/edit/test-group', dict( name="test-group", members=[User.objects.get(username="test").pk], - permissions=[HuePermission.objects.get(app='useradmin',action='access').pk], + permissions=[HuePermission.objects.get(app='useradmin', action='access').pk], save="Save" ), follow=True @@ -469,7 +515,7 @@ def test_group_permissions(self): # Check that we have access now response = c1.get('/useradmin/users') - assert_true(get_profile(test_user).has_hue_permission('access','useradmin')) + assert_true(get_profile(test_user).has_hue_permission('access', 'useradmin')) assert_true(b'Users' in response.content) # Make sure we can't modify permissions @@ -486,7 +532,7 @@ def test_group_permissions(self): follow=True ) assert_true(len(GroupPermission.objects.all()) == 0) - assert_false(get_profile(test_user).has_hue_permission('access','useradmin')) + assert_false(get_profile(test_user).has_hue_permission('access', 'useradmin')) # We should no longer have access to the app response = c1.get('/useradmin/users') @@ -552,7 +598,7 @@ def test_default_group(self): # Change the name of the default group, and try deleting again resets.append(useradmin.conf.DEFAULT_USER_GROUP.set_for_testing('new_default')) - response = c.post('/useradmin/groups/delete' , {'group_names': ['test_default']}) + response = c.post('/useradmin/groups/delete', {'group_names': ['test_default']}) assert_false(Group.objects.filter(name='test_default').exists()) assert_true(Group.objects.filter(name='new_default').exists()) finally: @@ -801,7 +847,7 @@ def test_user_admin(self): is_active=True ) ) - assert_true(b"You cannot remove" in response.content, "Shouldn't be able to remove the last superuser") + assert_true(b"You cannot remove" in response.content, "Shouldn't be able to remove the last superuser") # Shouldn't be able to delete oneself response = c.post('/useradmin/users/delete', {u'user_ids': [user.id], 'is_delete': True}) assert_true(b"You cannot remove yourself" in response.content, "Shouldn't be able to delete the last superuser") @@ -831,7 +877,9 @@ def test_user_admin(self): ) ) assert_equal( - [UserChangeForm.GENERIC_VALIDATION_ERROR], response.context[0]["form"]["password_old"].errors, "Should have complained about old password" + [UserChangeForm.GENERIC_VALIDATION_ERROR], + response.context[0]["form"]["password_old"].errors, + "Should have complained about old password" ) # Good now response = c.post('/useradmin/users/edit/test', dict( @@ -850,17 +898,26 @@ def test_user_admin(self): # Change it back! response = c.post('/hue/accounts/login/', dict(username="test", password="foo"), follow=True) - response = c.post('/useradmin/users/edit/test', dict(username="test", first_name="Tom", last_name="Tester", password1="test", password2="test", password_old="foo", is_active=True, is_superuser=True)) + response = c.post( + '/useradmin/users/edit/test', + dict( + username="test", first_name="Tom", last_name="Tester", password1="test", + password2="test", password_old="foo", is_active=True, is_superuser=True + ) + ) response = c.post('/hue/accounts/login/', dict(username="test", password="test"), follow=True) assert_true(User.objects.get(username="test").check_password("test")) - assert_true(make_logged_in_client(username = "test", password = "test"), "Check that we can still login.") + assert_true(make_logged_in_client(username="test", password="test"), "Check that we can still login.") # Check new user form for default group group = get_default_user_group() response = c.get('/useradmin/users/new') assert_true(response) - assert_true(('' % (group.id, group.name)) in (response.content if isinstance(response.content, str) else response.content.decode())) + assert_true( + ('' % (group.id, group.name)) in \ + (response.content if isinstance(response.content, str) else response.content.decode()) + ) # Create a new regular user (duplicate name) response = c.post('/useradmin/users/new', dict(username="test", password1="test", password2="test")) @@ -907,9 +964,9 @@ def test_user_admin(self): # Regular user should be able to modify oneself response = c_reg.post('/useradmin/users/edit/%s' % (FUNNY_NAME_QUOTED,), dict( - username = FUNNY_NAME, - first_name = "Hello", - is_active = True, + username=FUNNY_NAME, + first_name="Hello", + is_active=True, groups=[group.id for group in test_user.groups.all()] ), follow=True @@ -927,9 +984,9 @@ def test_user_admin(self): c_su = make_logged_in_client() # Inactivate FUNNY_NAME c_su.post('/useradmin/users/edit/%s' % (FUNNY_NAME_QUOTED,), dict( - username = FUNNY_NAME, - first_name = "Hello", - is_active = False) + username=FUNNY_NAME, + first_name="Hello", + is_active=False) ) # Now make sure FUNNY_NAME can't log back in response = c_reg.get('/useradmin/users/edit/%s' % (FUNNY_NAME_QUOTED,)) @@ -1016,8 +1073,12 @@ def test_list_for_autocomplete(self): # Now the autocomplete has access to all the users and groups c1 = make_logged_in_client('user_test_list_for_autocomplete', is_superuser=False, groupname='group_test_list_for_autocomplete') - c2_same_group = make_logged_in_client('user_test_list_for_autocomplete2', is_superuser=False, groupname='group_test_list_for_autocomplete') - c3_other_group = make_logged_in_client('user_test_list_for_autocomplete3', is_superuser=False, groupname='group_test_list_for_autocomplete_other_group') + c2_same_group = make_logged_in_client( + 'user_test_list_for_autocomplete2', is_superuser=False, groupname='group_test_list_for_autocomplete' + ) + c3_other_group = make_logged_in_client( + 'user_test_list_for_autocomplete3', is_superuser=False, groupname='group_test_list_for_autocomplete_other_group' + ) # c1 users should list only 'user_test_list_for_autocomplete2' and group should not list 'group_test_list_for_autocomplete_other_group' @@ -1066,9 +1127,13 @@ def test_list_for_autocomplete(self): content = json.loads(response.content) users = [smart_unicode(user['username']) for user in content['users']] - assert_equal([u'test', u'user_test_list_for_autocomplete', u'user_test_list_for_autocomplete2', u'user_test_list_for_autocomplete3'], users) + assert_equal( + [u'test', u'user_test_list_for_autocomplete', u'user_test_list_for_autocomplete2', u'user_test_list_for_autocomplete3'], users + ) - c5_autocomplete_filter_by_groupname = make_logged_in_client('user_doesnt_match_autocomplete_filter',is_superuser=False,groupname='group_test_list_for_autocomplete') + c5_autocomplete_filter_by_groupname = make_logged_in_client( + 'user_doesnt_match_autocomplete_filter', is_superuser=False, groupname='group_test_list_for_autocomplete' + ) # superuser should get all users & groups which match the autocomplete filter case insensitive response = c4_super_user.get('/desktop/api/users/autocomplete', {'include_myself': True, 'filter': 'Test_list_for_autocomplete'}) @@ -1077,7 +1142,7 @@ def test_list_for_autocomplete(self): users = [smart_unicode(user['username']) for user in content['users']] groups = [smart_unicode(user['name']) for user in content['groups']] - assert_equal([ u'user_test_list_for_autocomplete', u'user_test_list_for_autocomplete2', u'user_test_list_for_autocomplete3'], users) + assert_equal([u'user_test_list_for_autocomplete', u'user_test_list_for_autocomplete2', u'user_test_list_for_autocomplete3'], users) assert_equal([u'group_test_list_for_autocomplete', u'group_test_list_for_autocomplete_other_group'], groups) def test_language_preference(self): @@ -1112,9 +1177,15 @@ def test_edit_user_xss(self): ) ) if sys.version_info[0] < 3: - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + assert_true( + b'Select a valid choice. en-us><script>alert('Hacked')</script> '\ + b'is not one of the available choices.' in response.content + ) else: - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + assert_true( + b'Select a valid choice. en-us><script>alert('Hacked')</script> '\ + b'is not one of the available choices.' in response.content + ) # Hue 4 Admin response = edit_user.post('/useradmin/users/edit/admin', dict( username="admin", @@ -1137,9 +1208,15 @@ def test_edit_user_xss(self): ) ) if sys.version_info[0] < 3: - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + assert_true( + b'Select a valid choice. en-us><script>alert('Hacked')</script> '\ + b'is not one of the available choices.' in response.content + ) else: - assert_true(b'Select a valid choice. en-us><script>alert('Hacked')</script> is not one of the available choices.' in response.content) + assert_true( + b'Select a valid choice. en-us><script>alert('Hacked')</script> '\ + b'is not one of the available choices.' in response.content + ) # Hue 4, User with access to useradmin app response = edit_user.post('/useradmin/users/edit/edit_user', dict( username="edit_user", @@ -1191,7 +1268,10 @@ def test_ensure_home_directory(self): assert_false(cluster.fs.exists('/user/test2')) response = c.post('/useradmin/users/new', dict(username="test2", password1='test', password2='test')) assert_false(cluster.fs.exists('/user/test2')) - response = c.post('/useradmin/users/edit/%s' % "test2", dict(username="test2", password1='test', password2='test', password_old="test", ensure_home_directory=True)) + response = c.post( + '/useradmin/users/edit/%s' % "test2", + dict(username="test2", password1='test', password2='test', password_old="test", ensure_home_directory=True) + ) assert_true(cluster.fs.exists('/user/test2')) dir_stat = cluster.fs.stats('/user/test2') assert_equal('test2', dir_stat.user) @@ -1219,7 +1299,9 @@ def test_ensure_home_directory(self): if cluster.fs.exists('/user/test3'): cluster.fs.do_as_superuser(cluster.fs.rmtree, '/user/test3') assert_false(cluster.fs.exists('/user/test3')) - response = c.post('/useradmin/users/new', dict(username="test3@ad.sec.cloudera.com", password1='test', password2='test', ensure_home_directory=True)) + response = c.post( + '/useradmin/users/new', dict(username="test3@ad.sec.cloudera.com", password1='test', password2='test', ensure_home_directory=True) + ) assert_false(cluster.fs.exists('/user/test3@ad.sec.cloudera.com')) assert_true(cluster.fs.exists('/user/test3'))