detect large manifests and reduce size

CHANGELOG.md

@@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
   - Users can override this with the `--update-seedkit` flag in case AWS CodeSeeder has updated the SeedKit
 
 ### Fixes
+- adding in workaround for manifests whose char length is greater than SSM limit of 8192 k
 
 
 ## v3.1.2 (2024-01-24)

seedfarmer/mgmt/module_info.py

@@ -12,8 +12,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import json
 import logging
 import os
+import sys
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -23,7 +25,7 @@
 from seedfarmer import config
 from seedfarmer.services import _secrets_manager as secrets
 from seedfarmer.services import _ssm as ssm
-from seedfarmer.utils import generate_hash, generate_session_hash
+from seedfarmer.utils import generate_hash, generate_session_hash, remove_nulls
 
 _logger: logging.Logger = logging.getLogger(__name__)
 
@@ -538,7 +540,15 @@ def write_module_manifest(
     session: Session, optional
         The boto3.Session to use to for SSM Parameter queries, default None
     """
-    ssm.put_parameter(name=_manifest_key(deployment, group, module), obj=data, session=session)
+
+    # Temp fix until a larger persistence store is vetted
+    process_data = data
+    current_size = sys.getsizeof(json.dumps(data))
+    if current_size > 8191:
+        _logger.info("The manifest for %s-%s is %s, too large for SSM, reducing", group, module, current_size)
+        process_data = remove_nulls(data)
+        _logger.info("The size is now %s", sys.getsizeof(json.dumps(process_data)))
+    ssm.put_parameter(name=_manifest_key(deployment, group, module), obj=process_data, session=session)
 
 
 def write_deployspec(

seedfarmer/utils.py

@@ -15,7 +15,7 @@
 import hashlib
 import logging
 import os
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 import humps
 import yaml
@@ -183,3 +183,12 @@ def load_dotenv_files(root_path: str, env_files: List[str]) -> None:
         loaded_values.update(dotenv_values(dotenv_path, verbose=True))
 
     _logger.debug("Loaded environment variables: %s", loaded_values)
+
+
+def remove_nulls(payload: Dict[str, Any]) -> Dict[str, Any]:
+    if isinstance(payload, dict):
+        return {k: remove_nulls(v) for k, v in payload.items() if v is not None}
+    elif isinstance(payload, list):
+        return [remove_nulls(v) for v in payload]
+    else:
+        return payload

test/unit-test/test_mgmt_module_info.py

@@ -246,7 +246,8 @@ def test_write_module_manifest(aws_credentials, session, mocker):
     import seedfarmer.mgmt.module_info as mi
 
     mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True)
-    mi.write_module_manifest(deployment="myapp", group="test", module="mymodule", data={"Hey", "Yo"}, session=session)
+    payload = {"Hey", "Yo"}
+    mi.write_module_manifest(deployment="myapp", group="test", module="mymodule", data=dict.fromkeys(payload, 0), session=session)
 
 
 @pytest.mark.mgmt