diff --git a/.travis.yml b/.travis.yml index ce41bbd9..e920e18e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,11 @@ language: python python: - "2.7" install: - - "pip install funcsigs" - - "pip install pytest pytest-cov python-coveralls" + # Get rid of Travis' pytest 3.3.0, which somehow persists even though our + # requirements pin a newer version! + - "pip uninstall --yes pytest" - "python setup.py develop" script: - - "py.test tests --cov=src -vv" + - "pytest tests --cov=src -vv" after_success: - coveralls diff --git a/dev-requirements.txt b/dev-requirements.txt index c919e984..50ba3e08 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,8 @@ -coverage==4.5.1 +coverage==4.5.2 nose==1.3.7 -tox==2.9.1 -flake8==3.5.0 -pytest==3.4.0 +tox==3.7.0 +flake8==3.6.0 +pytest==4.1.1 +funcsigs==1.0.2 +pytest-cov==2.6.1 +coveralls==1.5.1 diff --git a/requirements.txt b/requirements.txt index b473e7d6..28997c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ jsonschema==2.6.0 lxml==4.2.1 path.py==11.0.1 pyquery==1.4.0 -pyxform==0.11.5 +pyxform==0.12.0 statistics==1.0.3.5 XlsxWriter==1.0.4 diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 8790e443..275ad0fa 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -23,7 +23,6 @@ class FormPack(object): - def __init__(self, versions=None, title='Submissions', id_string=None, default_version_id_key='__version__', strict_schema=False, @@ -193,19 +192,24 @@ def summr(v): @staticmethod def _combine_field_choices(old_field, new_field): - """ Update `new_field.choice` so that it contains everything from - `old_field.choice`. In the event of a conflict, `new_field.choice` - wins. If either field does not have a `choice` attribute, do - nothing + """ + Update `new_field.choice` so that it contains everything from + `old_field.choice`. In the event of a conflict, `new_field.choice` + wins. If either field does not have a `choice` attribute, do + nothing + + :param old_field: FormField + :param new_field: FormField + :return: FormField. Updated new_field """ try: old_choice = old_field.choice new_choice = new_field.choice + new_field.merge_choice(old_choice) except AttributeError: - return - combined_options = old_choice.options.copy() - combined_options.update(new_choice.options) - new_choice.options = combined_options + pass + + return new_field def get_fields_for_versions(self, versions=-1, data_types=None): @@ -273,7 +277,9 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # Because versions_desc are ordered from latest to oldest, # we use current field object as the old one and the one already # in position as the latest one. - self._combine_field_choices(field_object, latest_field_object) + new_object = self._combine_field_choices( + field_object, latest_field_object) + tmp2d[position[0]][position[1]] = new_object else: try: current_index_list = tmp2d[index] diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 18a81da4..86de9575 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -113,7 +113,6 @@ def parse_submissions(self, submissions): except KeyError: pass - def reset(self): """ Reset sections and indexes to initial values """ @@ -222,7 +221,6 @@ def get_fields_labels_tags_for_all_versions(self, len(field.value_names) ) - names = [name for name_list in name_lists for name in name_list] # add auto fields: @@ -429,8 +427,22 @@ def to_csv(self, submissions, sep=";", quote='"'): # if len(sections) > 1: # raise RuntimeError("CSV export does not support repeatable groups") + def escape_quote(value, quote): + ''' + According to https://www.ietf.org/rfc/rfc4180.txt, + + If double-quotes are used to enclose fields, then a + double-quote appearing inside a field must be escaped by + preceding it with another double quote. + + We will follow this convention by doubling `quote` wherever it + appears in `value`, regardless of what `quote` is. Perhaps this + is not the best idea. + ''' + return value.replace(quote, quote * 2) + def format_line(line, sep, quote): - line = [unicode(x) for x in line] + line = [escape_quote(unicode(x), quote) for x in line] return quote + (quote + sep + quote).join(line) + quote section, labels = sections[0] diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 0454f93e..6dae7790 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -704,12 +704,37 @@ def sum_frequencies(element): return stats + def merge_choice(self, choice): + """ + Update `new_field.choice` so that it contains everything from + `old_field.choice`. In the event of a conflict, `new_field.choice` + wins. If either field does not have a `choice` attribute, do + nothing + + :param choice: formpack.schema.datadef.FormChoice + """ + combined_options = choice.options.copy() + combined_options.update(self.choice.options) + self.choice.options = combined_options + + self._empty_result() + self.value_names = self.get_value_names() + + def _empty_result(self): + """ + Nothing to do here + """ + pass + class FormChoiceFieldWithMultipleSelect(FormChoiceField): """ Same as FormChoiceField, but you can select several answer """ def __init__(self, *args, **kwargs): super(FormChoiceFieldWithMultipleSelect, self).__init__(*args, **kwargs) + self._empty_result() + + def _empty_result(self): # reset empty result so it doesn't contain '0' self.empty_result = dict.fromkeys(self.empty_result, '') diff --git a/tests/fixtures/favorite_coffee/__init__.py b/tests/fixtures/favorite_coffee/__init__.py new file mode 100644 index 00000000..66ee9955 --- /dev/null +++ b/tests/fixtures/favorite_coffee/__init__.py @@ -0,0 +1,16 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + + +from ..load_fixture_json import load_fixture_json + +DATA = { + u'title': 'Favorite coffee', + u'id_string': 'favorite_coffee', + u'versions': [ + load_fixture_json('favorite_coffee/v1'), + load_fixture_json('favorite_coffee/v2') + ] +} diff --git a/tests/fixtures/favorite_coffee/v1.json b/tests/fixtures/favorite_coffee/v1.json new file mode 100644 index 00000000..7020b538 --- /dev/null +++ b/tests/fixtures/favorite_coffee/v1.json @@ -0,0 +1,65 @@ +{ + "id_string": "favorite_coffee", + "version": "fcv1", + "content": { + "choices": [ + { + "name": "french", + "label": [ + "French" + ], + "list_name": "al1hv46", + "order": 0 + }, + { + "name": "italian", + "label": [ + "Italian" + ], + "list_name": "al1hv46", + "order": 1 + }, + { + "name": "american", + "label": [ + "American" + ], + "list_name": "al1hv46", + "order": 2 + } + ], + "survey": [ + { + "select_from_list_name": "al1hv46", + "required": false, + "label": [ + "Favorite coffee type" + ], + "name": "favorite_coffee_type", + "type": "select_multiple" + }, + { + "required": false, + "type": "text", + "label": [ + "Brand of coffee machine" + ], + "name": "brand_of_coffee_machine" + } + ] + }, + "submissions": [ + { + "brand_of_coffee_machine": "Breville", + "favorite_coffee_type": "french italian" + }, + { + "brand_of_coffee_machine": "DeLonghi", + "favorite_coffee_type": "italian" + }, + { + "brand_of_coffee_machine": "Nespresso", + "favorite_coffee_type": "american" + } + ] +} diff --git a/tests/fixtures/favorite_coffee/v2.json b/tests/fixtures/favorite_coffee/v2.json new file mode 100644 index 00000000..f09add3f --- /dev/null +++ b/tests/fixtures/favorite_coffee/v2.json @@ -0,0 +1,61 @@ +{ + "id_string": "favorite_coffee", + "version": "fcv2", + "content": { + "choices": [ + { + "name": "french", + "label": [ + "French" + ], + "list_name": "al1hv46", + "order": 0 + }, + { + "name": "american", + "label": [ + "American" + ], + "list_name": "al1hv46", + "order": 1 + }, + { + "name": "british", + "label": [ + "British" + ], + "list_name": "al1hv46", + "order": 2 + } + ], + "survey": [ + { + "select_from_list_name": "al1hv46", + "required": false, + "label": [ + "Favorite coffee type" + ], + "name": "favorite_coffee_type", + "type": "select_multiple" + }, + { + "required": false, + "type": "text", + "label": [ + "Brand of coffee machine" + ], + "name": "brand_of_coffee_machine" + } + ] + }, + "submissions": [ + { + "brand_of_coffee_machine": "Saico", + "favorite_coffee_type": "french" + }, + { + "brand_of_coffee_machine": "Keurig", + "favorite_coffee_type": "american british" + } + ] +} diff --git a/tests/fixtures/quotes_newlines_and_long_urls/__init__.py b/tests/fixtures/quotes_newlines_and_long_urls/__init__.py new file mode 100644 index 00000000..29d6c21c --- /dev/null +++ b/tests/fixtures/quotes_newlines_and_long_urls/__init__.py @@ -0,0 +1,101 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + +''' +Quotes, newlines, and long URLs: oh, my! + +Double quotation marks need to be escaped (by doubling them!) in CSV exports, + e.g. `Introduce excerpts with "` would become `Introduce excerpts with ""`. + +Excel does not tolerate hyperlinks with URLs longer than 255 characters. Not +that we ever explicitly create Excel hyperlinks, but +https://github.com/jmcnamara/XlsxWriter tries to be helpful and add them +automatically. Excessively long URLs should just be written as strings. +''' + +DATA = { + 'title': 'Quotes, newlines, and long URLs', + 'id_string': 'quotes_newlines_and_long_urls', + 'versions': [ + { + "version": "first", + "content": { + "choices": [ + { + "label": ["yes"], + "list_name": "yes_no", + "name": "yes", + }, + { + "label": ["no"], + "list_name": "yes_no", + "name": "no", + }, + ], + "survey": [ + { + "required": False, + "appearance": "multiline", + "name": "Enter_some_long_text_and_linebreaks_here", + "label": [ + "Enter some long text with \" and linebreaks here" + ], + "type": "text", + }, + { + "select_from_list_name": "yes_no", + "required": False, + "name": "Some_other_question", + "label": ["Some other question"], + "type": "select_one", + }, + ], + }, + 'submissions': [ + { + "Enter_some_long_text_and_linebreaks_here": + "Check out this URL I found:\n" + "https://now.read.this/?Never%20forget%20that%20you%20" + "are%20one%20of%20a%20kind.%20Never%20forget%20that%20" + "if%20there%20weren%27t%20any%20need%20for%20you%20in" + "%20all%20your%20uniqueness%20to%20be%20on%20this%20" + "earth%2C%20you%20wouldn%27t%20be%20here%20in%20the%20" + "first%20place.%20And%20never%20forget%2C%20no%20" + "matter%20how%20overwhelming%20life%27s%20challenges" + "%20and%20problems%20seem%20to%20be%2C%20that%20one%20" + "person%20can%20make%20a%20difference%20in%20the%20" + "world.%20In%20fact%2C%20it%20is%20always%20because%20" + "of%20one%20person%20that%20all%20the%20changes%20that" + "%20matter%20in%20the%20world%20come%20about.%20So%20" + "be%20that%20one%20person.", + "Some_other_question": "yes", + }, + # Thanks to @tinok for the whimisical sample data below + { + "Enter_some_long_text_and_linebreaks_here": + "Hi, my name is Roger.\"\n\nI like to enter quotes " + "randomly and follow them with new lines.", + "Some_other_question": "yes", + }, + { + "Enter_some_long_text_and_linebreaks_here": + "This one has no linebreaks", + "Some_other_question": "no", + }, + { + "Enter_some_long_text_and_linebreaks_here": + "This\nis\nnot\na Haiku", + "Some_other_question": "yes", + }, + { + "Enter_some_long_text_and_linebreaks_here": + "\"Hands up!\" He yelled.\nWhy?\"\nShe couldn't " + "understand anything.", + "Some_other_question": "yes", + }, + ], + }, + ], +} diff --git a/tests/test_exports.py b/tests/test_exports.py index 58657dba..3a9898f9 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -191,7 +191,6 @@ def test_export_with_choice_lists(self): '', 'traditionnel']]) - def test_headers_of_group_exports(self): title, schemas, submissions = build_fixture('grouped_questions') fp = FormPack(schemas, title) @@ -1108,6 +1107,43 @@ def test_csv(self): self.assertTextEqual(csv_data, expected) + def test_csv_quote_escaping(self): + title, schemas, submissions = build_fixture( + 'quotes_newlines_and_long_urls') + fp = FormPack(schemas, title) + lss = list(submissions) + csv_lines = list(fp.export().to_csv(submissions)) + expected_lines = [] + expected_lines.append( + '"Enter_some_long_text_and_linebreaks_here";' + '"Some_other_question"' + ) + expected_lines.append( + '"Check out this URL I found:\nhttps://now.read.this/?Never%20forg' + 'et%20that%20you%20are%20one%20of%20a%20kind.%20Never%20forget%20t' + 'hat%20if%20there%20weren%27t%20any%20need%20for%20you%20in%20all%' + '20your%20uniqueness%20to%20be%20on%20this%20earth%2C%20you%20woul' + 'dn%27t%20be%20here%20in%20the%20first%20place.%20And%20never%20fo' + 'rget%2C%20no%20matter%20how%20overwhelming%20life%27s%20challenge' + 's%20and%20problems%20seem%20to%20be%2C%20that%20one%20person%20ca' + 'n%20make%20a%20difference%20in%20the%20world.%20In%20fact%2C%20it' + '%20is%20always%20because%20of%20one%20person%20that%20all%20the%2' + '0changes%20that%20matter%20in%20the%20world%20come%20about.%20So%' + '20be%20that%20one%20person.";"yes"' + ) + expected_lines.append( + '"Hi, my name is Roger.""\n\nI like to enter quotes randomly and ' + 'follow them with new lines.";"yes"' + ) + expected_lines.append('"This one has no linebreaks";"no"') + expected_lines.append('"This\nis\nnot\na Haiku";"yes"') + expected_lines.append( + '"""Hands up!"" He yelled.\nWhy?""\n' + '''She couldn't understand anything.";"yes"''' + ) + + self.assertListEqual(csv_lines, expected_lines) + def test_csv_with_tag_headers(self): title, schemas, submissions = build_fixture('dietary_needs') fp = FormPack(schemas, title) @@ -1746,3 +1782,34 @@ def test_untranslated_spss_labels(self): assert actual.read() == expected.read() zipped.close() raw_zip.close() + + def test_select_multiple_with_different_options_in_multiple_versions(self): + title, schemas, submissions = build_fixture('favorite_coffee') + fp = FormPack(schemas, title) + self.assertEqual(len(fp.versions), 2) + + export = fp.export(versions=fp.versions.keys()).to_dict(submissions) + + headers = export['Favorite coffee']['fields'] + self.assertListEqual(headers, [ + 'favorite_coffee_type', + 'favorite_coffee_type/french', + 'favorite_coffee_type/italian', + 'favorite_coffee_type/american', + 'favorite_coffee_type/british', + 'brand_of_coffee_machine' + ]) + + # Check length of each row + for row in export['Favorite coffee']['data']: + self.assertEqual(len(headers), len(row)) + + # Ensure latest submissions is not shifted + self.assertListEqual(export['Favorite coffee']['data'][-1], [ + 'american british', + '0', + '0', + '1', + '1', + 'Keurig' + ])