Allow specifying a new strict option which will allow list items in a…

…ny order (taverntesting#604)

smoke.bash

@@ -2,7 +2,7 @@
 
 set -ex
 
-PYVER=38
+PYVER=39
 
 tox -c tox.ini        \
   -e py${PYVER}flakes \

tavern/_plugins/mqtt/response.py

@@ -59,7 +59,7 @@ def _await_response(self):
         timeout = self.expected.get("timeout", 1)
 
         test_strictness = self.test_block_config["strict"]
-        block_strictness = test_strictness.setting_for("json").is_on()
+        block_strictness = test_strictness.setting_for("json")
 
         expected_payload, expect_json_payload = self._get_payload_vals()
 

tavern/_plugins/rest/response.py

@@ -232,5 +232,5 @@ def _validate_block(self, blockname, block):
         logger.debug("Validating response %s against %s", blockname, expected_block)
 
         test_strictness = self.test_block_config["strict"]
-        block_strictness = test_strictness.setting_for(blockname).is_on()
+        block_strictness = test_strictness.setting_for(blockname)
         self.recurse_check_key_match(expected_block, block, blockname, block_strictness)

tavern/response/base.py

@@ -56,6 +56,7 @@ def recurse_check_key_match(self, expected_block, block, blockname, strict):
             Optionally use a validation library too
 
         Args:
+            strict: strictness setting for this block
             expected_block (dict): expected data
             block (dict): actual data
             blockname (str): 'name' of this block (params, mqtt, etc) for error messages

tavern/util/dict_util.py

@@ -17,6 +17,7 @@
 
 from . import exceptions
 from .formatted_str import FormattedString
+from .strict_util import StrictSetting, extract_strict_setting
 
 logger = logging.getLogger(__name__)
 
@@ -350,7 +351,7 @@ def check_keys_match_recursive(expected_val, actual_val, keys, strict=True):
         KeyMismatchError: expected_val and actual_val did not match
     """
 
-    # pylint: disable=too-many-locals,too-many-statements
+    # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks
 
     def full_err():
         """Get error in the format:
@@ -395,6 +396,8 @@ def _format_err(which):
             issubclass(actual_type, type(expected_val))
         )
 
+    strict, strict_setting = extract_strict_setting(strict)
+
     try:
         assert actual_val == expected_val
     except AssertionError as e:
@@ -455,7 +458,7 @@ def _format_err(which):
             for key in to_recurse:
                 try:
                     check_keys_match_recursive(
-                        expected_val[key], actual_val[key], keys + [key], strict
+                        expected_val[key], actual_val[key], keys + [key], strict_setting
                     )
                 except KeyError:
                     logger.debug(
@@ -489,7 +492,7 @@ def _format_err(which):
                         # Found one - check if it matches
                         try:
                             check_keys_match_recursive(
-                                e_val, current_response_val, keys + [i], strict
+                                e_val, current_response_val, keys + [i], strict_setting
                             )
                         except exceptions.KeyMismatchError:
                             # Doesn't match what we're looking for
@@ -500,6 +503,8 @@ def _format_err(which):
                             )
                         else:
                             logger.debug("'%s' present in response", e_val)
+                            if strict_setting == StrictSetting.LIST_ANY_ORDER:
+                                actual_iter = iter(actual_val)
                             break
 
                 if missing:
@@ -517,11 +522,13 @@ def _format_err(which):
 
                 for i, (e_val, a_val) in enumerate(zip(expected_val, actual_val)):
                     try:
-                        check_keys_match_recursive(e_val, a_val, keys + [i], strict)
+                        check_keys_match_recursive(
+                            e_val, a_val, keys + [i], strict_setting
+                        )
                     except exceptions.KeyMismatchError as sub_e:
-                        # This should _ALWAYS_ raise an error, but it will be more
-                        # obvious where the error came from (in python 3 at least)
-                        # and will take ANYTHING into account
+                        # This should _ALWAYS_ raise an error (unless the reason it didn't match was the
+                        # 'anything' sentinel), but it will be more obvious where the error came from
+                        # (in python 3 at least) and will take ANYTHING into account
                         raise sub_e from e
         elif expected_val is ANYTHING:
             logger.debug("Actual value = '%s' - matches !anything", actual_val)

tavern/util/strict_util.py

@@ -1,52 +1,65 @@
 from distutils.util import strtobool
 import enum
+import logging
 import re
 
 import attr
 
 from tavern.util import exceptions
 
+logger = logging.getLogger(__name__)
 
-class _StrictSetting(enum.Enum):
+
+class StrictSetting(enum.Enum):
     ON = 1
     OFF = 2
     UNSET = 3
+    LIST_ANY_ORDER = 4
 
 
 valid_keys = ["json", "headers", "redirect_query_params"]
 
+valid_switches = ["on", "off", "list_any_order"]
+
 
-def setting_factory(str_setting):
+def strict_setting_factory(str_setting):
     """Converts from cmdline/setting file to an enum"""
     if str_setting is None:
-        return _StrictSetting.UNSET
+        return StrictSetting.UNSET
     else:
+        if str_setting == "list_any_order":
+            return StrictSetting.LIST_ANY_ORDER
+
         parsed = strtobool(str_setting)
 
         if parsed:
-            return _StrictSetting.ON
+            return StrictSetting.ON
         else:
-            return _StrictSetting.OFF
+            return StrictSetting.OFF
 
 
 @attr.s(frozen=True)
-class _StrictOption:
+class StrictOption:
     section = attr.ib(type=str)
-    setting = attr.ib(type=_StrictSetting)
+    setting = attr.ib(type=StrictSetting)
 
     def is_on(self):
         if self.section == "json":
             # Must be specifically disabled for response body
-            return self.setting != _StrictSetting.OFF
+            return self.setting not in [StrictSetting.OFF, StrictSetting.LIST_ANY_ORDER]
         else:
             # Off by default for everything else
-            return self.setting == _StrictSetting.ON
+            return self.setting in [StrictSetting.ON]
 
 
 def validate_and_parse_option(key):
-    regex = r"(?P<section>{})(:(?P<setting>on|off))?".format("|".join(valid_keys))
+    regex = re.compile(
+        "(?P<section>{sections})(:(?P<setting>{switches}))?".format(
+            sections="|".join(valid_keys), switches="|".join(valid_switches)
+        )
+    )
 
-    match = re.fullmatch(regex, key)
+    match = regex.fullmatch(key)
 
     if not match:
         raise exceptions.InvalidConfigurationException(
@@ -56,15 +69,21 @@ def validate_and_parse_option(key):
         )
 
     as_dict = match.groupdict()
-    return _StrictOption(as_dict["section"], setting_factory(as_dict["setting"]))
+
+    if as_dict["section"] != "json" and as_dict["setting"] == "list_any_order":
+        logger.warning(
+            "Using 'list_any_order' key outside of 'json' section has no meaning"
+        )
+
+    return StrictOption(as_dict["section"], strict_setting_factory(as_dict["setting"]))
 
 
 @attr.s(frozen=True)
 class StrictLevel:
-    json = attr.ib(default=_StrictOption("json", setting_factory(None)))
-    headers = attr.ib(default=_StrictOption("headers", setting_factory(None)))
+    json = attr.ib(default=StrictOption("json", strict_setting_factory(None)))
+    headers = attr.ib(default=StrictOption("headers", strict_setting_factory(None)))
     redirect_query_params = attr.ib(
-        default=_StrictOption("redirect_query_params", setting_factory(None))
+        default=StrictOption("redirect_query_params", strict_setting_factory(None))
     )
 
     @classmethod
@@ -96,3 +115,26 @@ def all_on(cls):
     @classmethod
     def all_off(cls):
         return cls.from_options([i + ":off" for i in valid_keys])
+
+
+def extract_strict_setting(strict):
+    """Takes either a bool, StrictOption, or a StrictSetting and return the bool representation and StrictSetting representation"""
+    if isinstance(strict, StrictSetting):
+        strict_setting = strict
+        strict = strict == StrictSetting.ON
+    elif isinstance(strict, StrictOption):
+        strict_setting = strict.setting
+        strict = strict.is_on()
+    elif isinstance(strict, bool):
+        strict_setting = strict_setting_factory(str(strict))
+    elif strict is None:
+        strict = False
+        strict_setting = strict_setting_factory("false")
+    else:
+        raise exceptions.InvalidConfigurationException(
+            "Unable to parse strict setting '{}' of type '{}'".format(
+                strict, type(strict)
+            )
+        )
+
+    return strict, strict_setting

tests/integration/test_strict_key_checks.tavern.yaml

@@ -319,7 +319,7 @@ stages:
       url: "{host}/fake_list"
     response:
       status_code: 200
-#      Use new syntax
+      #      Use new syntax
       strict:
         - json:off
       json:
@@ -422,6 +422,95 @@ stages:
 
 ---
 
+test_name: Test matching any order on json
+
+strict:
+  - json:list_any_order
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/fake_list"
+    response:
+      status_code: 200
+      json:
+        - 2
+        - c
+        - a
+        - -3.0
+        - 1
+
+---
+
+test_name: Test matching any order on json
+
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/fake_list"
+    response:
+      strict:
+        - json:list_any_order
+      status_code: 200
+      json:
+        - 2
+        - c
+        - a
+        - -3.0
+        - 1
+
+---
+
+test_name: Test matching any order on json nested
+
+strict:
+  - json:list_any_order
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/nested_list"
+    response:
+      status_code: 200
+      json:
+        top:
+        - b
+        - key: val
+        - a
+
+---
+
+test_name: Test matching any order on json nested
+
+
+includes:
+  - !include common.yaml
+
+stages:
+  - name: match some things in list, in any order
+    request:
+      url: "{host}/nested_list"
+    response:
+      strict:
+        - json:list_any_order
+      status_code: 200
+      json:
+        top:
+        - b
+        - key: val
+
+---
+
 test_name: Test non-strict key matching in one stage does not leak over to the next
 
 _xfail: run

tests/unit/test_helpers.py

@@ -21,7 +21,7 @@
 from tavern.util.loader import ForceIncludeToken
 from tavern.util.strict_util import (
     StrictLevel,
-    _StrictSetting,
+    StrictSetting,
     validate_and_parse_option,
 )
 
@@ -393,18 +393,18 @@ def test_defaults(self, section):
     def test_set_on(self, section):
         level = StrictLevel.from_options([section + ":on"])
 
-        assert level.setting_for(section).setting == _StrictSetting.ON
+        assert level.setting_for(section).setting == StrictSetting.ON
         assert level.setting_for(section).is_on()
 
     @pytest.mark.parametrize("section", ["json", "headers", "redirect_query_params"])
     def test_set_off(self, section):
         level = StrictLevel.from_options([section + ":off"])
 
-        assert level.setting_for(section).setting == _StrictSetting.OFF
+        assert level.setting_for(section).setting == StrictSetting.OFF
         assert not level.setting_for(section).is_on()
 
     @pytest.mark.parametrize("section", ["json", "headers", "redirect_query_params"])
     def test_unset(self, section):
         level = StrictLevel.from_options([section])
 
-        assert level.setting_for(section).setting == _StrictSetting.UNSET
+        assert level.setting_for(section).setting == StrictSetting.UNSET

tests/unit/test_strict_util.py

@@ -0,0 +1,27 @@
+import pytest
+
+from tavern.util.strict_util import StrictOption, StrictSetting, extract_strict_setting
+
+
+@pytest.mark.parametrize(
+    "strict", [True, StrictSetting.ON, StrictOption("json", StrictSetting.ON)]
+)
+def test_extract_strict_setting_true(strict):
+    as_bool, as_setting = extract_strict_setting(strict)
+    assert as_bool is True
+
+
+@pytest.mark.parametrize(
+    "strict",
+    [
+        False,
+        StrictSetting.OFF,
+        StrictSetting.LIST_ANY_ORDER,
+        StrictSetting.UNSET,
+        StrictOption("json", StrictSetting.OFF),
+        None,
+    ],
+)
+def test_extract_strict_setting_false(strict):
+    as_bool, as_setting = extract_strict_setting(strict)
+    assert as_bool is False