Merge remote-tracking branch 'origin/master' into 185-autoreport-cras…

…hes-with-non-numeric-in-numeric-field

.travis.yml

@@ -3,10 +3,11 @@ language: python
 python:
   - "2.7"
 install:
-  - "pip install funcsigs"
-  - "pip install pytest pytest-cov python-coveralls"
+  # Get rid of Travis' pytest 3.3.0, which somehow persists even though our
+  # requirements pin a newer version!
+  - "pip uninstall --yes pytest"
   - "python setup.py develop"
 script:
-  - "py.test tests --cov=src -vv"
+  - "pytest tests --cov=src -vv"
 after_success:
   - coveralls

dev-requirements.txt

@@ -1,5 +1,8 @@
-coverage==4.5.1
+coverage==4.5.2
 nose==1.3.7
-tox==2.9.1
-flake8==3.5.0
-pytest==3.4.0
+tox==3.7.0
+flake8==3.6.0
+pytest==4.1.1
+funcsigs==1.0.2
+pytest-cov==2.6.1
+coveralls==1.5.1

requirements.txt

@@ -4,6 +4,6 @@ jsonschema==2.6.0
 lxml==4.2.1
 path.py==11.0.1
 pyquery==1.4.0
-pyxform==0.11.5
+pyxform==0.12.0
 statistics==1.0.3.5
 XlsxWriter==1.0.4

src/formpack/pack.py

@@ -23,7 +23,6 @@
 
 class FormPack(object):
 
-
     def __init__(self, versions=None, title='Submissions', id_string=None,
                  default_version_id_key='__version__',
                  strict_schema=False,
@@ -193,19 +192,24 @@ def summr(v):
 
     @staticmethod
     def _combine_field_choices(old_field, new_field):
-        """ Update `new_field.choice` so that it contains everything from
-            `old_field.choice`. In the event of a conflict, `new_field.choice`
-            wins. If either field does not have a `choice` attribute, do
-            nothing
+        """
+        Update `new_field.choice` so that it contains everything from
+        `old_field.choice`. In the event of a conflict, `new_field.choice`
+        wins. If either field does not have a `choice` attribute, do
+        nothing
+
+        :param old_field: FormField
+        :param new_field: FormField
+        :return: FormField. Updated new_field
         """
         try:
             old_choice = old_field.choice
             new_choice = new_field.choice
+            new_field.merge_choice(old_choice)
         except AttributeError:
-            return
-        combined_options = old_choice.options.copy()
-        combined_options.update(new_choice.options)
-        new_choice.options = combined_options
+            pass
+
+        return new_field
 
     def get_fields_for_versions(self, versions=-1, data_types=None):
 
@@ -273,7 +277,9 @@ def get_fields_for_versions(self, versions=-1, data_types=None):
                             # Because versions_desc are ordered from latest to oldest,
                             # we use current field object as the old one and the one already
                             # in position as the latest one.
-                            self._combine_field_choices(field_object, latest_field_object)
+                            new_object = self._combine_field_choices(
+                                field_object, latest_field_object)
+                            tmp2d[position[0]][position[1]] = new_object
                         else:
                             try:
                                 current_index_list = tmp2d[index]

src/formpack/reporting/export.py

@@ -113,7 +113,6 @@ def parse_submissions(self, submissions):
             except KeyError:
                 pass
 
-
     def reset(self):
         """ Reset sections and indexes to initial values """
 
@@ -222,7 +221,6 @@ def get_fields_labels_tags_for_all_versions(self,
                         len(field.value_names)
                 )
 
-
             names = [name for name_list in name_lists for name in name_list]
 
             # add auto fields:
@@ -429,8 +427,22 @@ def to_csv(self, submissions, sep=";", quote='"'):
         # if len(sections) > 1:
         #     raise RuntimeError("CSV export does not support repeatable groups")
 
+        def escape_quote(value, quote):
+            '''
+                According to https://www.ietf.org/rfc/rfc4180.txt,
+
+                    If double-quotes are used to enclose fields, then a
+                    double-quote appearing inside a field must be escaped by
+                    preceding it with another double quote.
+
+                We will follow this convention by doubling `quote` wherever it
+                appears in `value`, regardless of what `quote` is. Perhaps this
+                is not the best idea.
+            '''
+            return value.replace(quote, quote * 2)
+
         def format_line(line, sep, quote):
-            line = [unicode(x) for x in line]
+            line = [escape_quote(unicode(x), quote) for x in line]
             return quote + (quote + sep + quote).join(line) + quote
 
         section, labels = sections[0]

src/formpack/schema/fields.py

@@ -704,12 +704,37 @@ def sum_frequencies(element):
 
         return stats
 
+    def merge_choice(self, choice):
+        """
+        Update `new_field.choice` so that it contains everything from
+        `old_field.choice`. In the event of a conflict, `new_field.choice`
+        wins. If either field does not have a `choice` attribute, do
+        nothing
+
+        :param choice: formpack.schema.datadef.FormChoice
+        """
+        combined_options = choice.options.copy()
+        combined_options.update(self.choice.options)
+        self.choice.options = combined_options
+
+        self._empty_result()
+        self.value_names = self.get_value_names()
+
+    def _empty_result(self):
+        """
+        Nothing to do here
+        """
+        pass
+
 
 class FormChoiceFieldWithMultipleSelect(FormChoiceField):
     """  Same as FormChoiceField, but you can select several answer """
 
     def __init__(self, *args, **kwargs):
         super(FormChoiceFieldWithMultipleSelect, self).__init__(*args, **kwargs)
+        self._empty_result()
+
+    def _empty_result(self):
         # reset empty result so it doesn't contain '0'
         self.empty_result = dict.fromkeys(self.empty_result, '')
 

tests/fixtures/favorite_coffee/__init__.py

@@ -0,0 +1,16 @@
+# coding: utf-8
+
+from __future__ import (unicode_literals, print_function,
+                        absolute_import, division)
+
+
+from ..load_fixture_json import load_fixture_json
+
+DATA = {
+    u'title': 'Favorite coffee',
+    u'id_string': 'favorite_coffee',
+    u'versions': [
+        load_fixture_json('favorite_coffee/v1'),
+        load_fixture_json('favorite_coffee/v2')
+    ]
+}

tests/fixtures/favorite_coffee/v1.json

@@ -0,0 +1,65 @@
+{
+  "id_string": "favorite_coffee",
+  "version": "fcv1",
+  "content": {
+    "choices": [
+      {
+        "name": "french",
+        "label": [
+          "French"
+        ],
+        "list_name": "al1hv46",
+        "order": 0
+      },
+      {
+        "name": "italian",
+        "label": [
+          "Italian"
+        ],
+        "list_name": "al1hv46",
+        "order": 1
+      },
+      {
+        "name": "american",
+        "label": [
+          "American"
+        ],
+        "list_name": "al1hv46",
+        "order": 2
+      }
+    ],
+    "survey": [
+      {
+        "select_from_list_name": "al1hv46",
+        "required": false,
+        "label": [
+          "Favorite coffee type"
+        ],
+        "name": "favorite_coffee_type",
+        "type": "select_multiple"
+      },
+      {
+        "required": false,
+        "type": "text",
+        "label": [
+          "Brand of coffee machine"
+        ],
+        "name": "brand_of_coffee_machine"
+      }
+    ]
+  },
+  "submissions": [
+    {
+      "brand_of_coffee_machine": "Breville",
+      "favorite_coffee_type": "french italian"
+    },
+    {
+      "brand_of_coffee_machine": "DeLonghi",
+      "favorite_coffee_type": "italian"
+    },
+    {
+      "brand_of_coffee_machine": "Nespresso",
+      "favorite_coffee_type": "american"
+    }
+  ]
+}

tests/fixtures/favorite_coffee/v2.json

@@ -0,0 +1,61 @@
+{
+  "id_string": "favorite_coffee",
+  "version": "fcv2",
+  "content": {
+    "choices": [
+      {
+        "name": "french",
+        "label": [
+          "French"
+        ],
+        "list_name": "al1hv46",
+        "order": 0
+      },
+      {
+        "name": "american",
+        "label": [
+          "American"
+        ],
+        "list_name": "al1hv46",
+        "order": 1
+      },
+      {
+        "name": "british",
+        "label": [
+          "British"
+        ],
+        "list_name": "al1hv46",
+        "order": 2
+      }
+    ],
+    "survey": [
+      {
+        "select_from_list_name": "al1hv46",
+        "required": false,
+        "label": [
+          "Favorite coffee type"
+        ],
+        "name": "favorite_coffee_type",
+        "type": "select_multiple"
+      },
+      {
+        "required": false,
+        "type": "text",
+        "label": [
+          "Brand of coffee machine"
+        ],
+        "name": "brand_of_coffee_machine"
+      }
+    ]
+  },
+  "submissions": [
+    {
+      "brand_of_coffee_machine": "Saico",
+      "favorite_coffee_type": "french"
+    },
+    {
+      "brand_of_coffee_machine": "Keurig",
+      "favorite_coffee_type": "american british"
+    }
+  ]
+}