From 07ff9846f3a5c6502dd5e56d65fda37f96ecdb49 Mon Sep 17 00:00:00 2001 From: alxtkr77 <3098237+alxtkr77@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:59:04 +0300 Subject: [PATCH] Raise an error on too long string attributes (#123) --- tests/test_client.py | 11 ++++++++-- v3io/dataplane/kv_large_string.py | 34 ------------------------------- v3io/dataplane/output.py | 15 ++++++-------- v3io/dataplane/request.py | 30 ++++++++++++++++----------- 4 files changed, 33 insertions(+), 57 deletions(-) delete mode 100644 v3io/dataplane/kv_large_string.py diff --git a/tests/test_client.py b/tests/test_client.py index 8224be0..73646ff 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -27,7 +27,6 @@ import v3io.dataplane.output import v3io.dataplane.response import v3io.logger -from v3io.dataplane.kv_large_string import LARGE_STRING_MIN_SIZE class Test(unittest.TestCase): @@ -432,7 +431,6 @@ def _get_float_array(): "array_with_ints": _get_int_array(), "array_with_floats": _get_float_array(), "now": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc), - "large_string": "a" * 10 * LARGE_STRING_MIN_SIZE, } } @@ -448,6 +446,15 @@ def _get_float_array(): for key in item[item_key]: self._compare_item_types(item[item_key][key], response.output.item[key]) + item = {item_key: {"large_string": "a" * 61200}} + try: + self._client.kv.put( + container=self._container, table_path=self._path, key=item_key, attributes=item[item_key] + ) + self.fail("Large string should have raised an exception") + except AttributeError: + pass + def test_kv(self): items = { "bob": {"age": 42, "feature": "mustache"}, diff --git a/v3io/dataplane/kv_large_string.py b/v3io/dataplane/kv_large_string.py deleted file mode 100644 index c1186ff..0000000 --- a/v3io/dataplane/kv_large_string.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2024 Iguazio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import zlib - -LARGE_STRING_MIN_SIZE = 60000 - -prefix = b"_v3io_large_string" - - -def is_large_bstring(attribute_value): - return attribute_value[: len(prefix)] == prefix - - -def large_bstring_to_string(attribute_value): - compressed_value = attribute_value[len(prefix) :] - return zlib.decompress(compressed_value).decode("utf-8") - - -def string_to_large_bstring(attribute_value): - bvalue = zlib.compress(attribute_value.encode("utf-8")) - return prefix + bvalue diff --git a/v3io/dataplane/output.py b/v3io/dataplane/output.py index 9721036..9b81942 100644 --- a/v3io/dataplane/output.py +++ b/v3io/dataplane/output.py @@ -18,7 +18,6 @@ import v3io.dataplane.kv_array import v3io.dataplane.kv_timestamp -from v3io.dataplane.kv_large_string import is_large_bstring, large_bstring_to_string class Output(object): @@ -34,14 +33,12 @@ def _decode_typed_attributes(self, typed_attributes): decoded_attribute = float(attribute_value) elif attribute_type == "B": decoded_attribute = base64.b64decode(attribute_value) - if is_large_bstring(decoded_attribute): - decoded_attribute = large_bstring_to_string(decoded_attribute) - else: - # try to decode as an array - try: - decoded_attribute = v3io.dataplane.kv_array.decode(decoded_attribute) - except BaseException: - pass + + # try to decode as an array + try: + decoded_attribute = v3io.dataplane.kv_array.decode(decoded_attribute) + except BaseException: + pass elif attribute_type == "S": if type(attribute_value) in [float, int]: diff --git a/v3io/dataplane/request.py b/v3io/dataplane/request.py index 673196b..1983c11 100644 --- a/v3io/dataplane/request.py +++ b/v3io/dataplane/request.py @@ -29,10 +29,6 @@ import v3io.common.helpers import v3io.dataplane.kv_array import v3io.dataplane.kv_timestamp -from v3io.dataplane.kv_large_string import ( - LARGE_STRING_MIN_SIZE, - string_to_large_bstring, -) # # Request @@ -418,19 +414,29 @@ def _to_base64(input): def _dict_to_typed_attributes(d): typed_attributes = {} - + max_string_length = 61199 for key, value in future.utils.viewitems(d): attribute_type = type(value) type_value = None - if isinstance(value, future.utils.text_type) or isinstance(value, future.utils.string_types): + if isinstance(value, future.utils.text_type): + type_key = "S" + type_value = value + if len(value) > max_string_length: + raise AttributeError( + "Attribute {0} is too long({1} bytes) when max is {2} bytes".format( + key, len(value), max_string_length + ) + ) + elif isinstance(value, future.utils.string_types): + type_key = "S" type_value = str(value) - if len(value) > LARGE_STRING_MIN_SIZE: - type_key = "B" - type_value = string_to_large_bstring(type_value) - type_value = base64.b64encode(type_value) - else: - type_key = "S" + if len(type_value) > max_string_length: + raise AttributeError( + "Attribute {0} is too long({1} bytes) when max is {2} bytes".format( + key, len(value), max_string_length + ) + ) elif attribute_type in [int, float]: type_key = "N" type_value = str(value)