diff --git a/emission/net/api/usercache.py b/emission/net/api/usercache.py index b2066b243..c16e62acc 100644 --- a/emission/net/api/usercache.py +++ b/emission/net/api/usercache.py @@ -29,6 +29,19 @@ def sync_server_to_phone(uuid): # logging.debug("retrievedData = %s" % retrievedData) return retrievedData +def _remove_dots(entry_doc): + for key in entry_doc: + # print(f"Checking {key=}") + if isinstance(entry_doc[key], dict): + # print(f"Found dict for {key=}, recursing") + _remove_dots(entry_doc[key]) + if '.' in key: + munged_key = key.replace(".", "_") + logging.info(f"Found {key=} with dot, munged to {munged_key=}") + # Get and delete in one swoop + # https://stackoverflow.com/a/11277439 + entry_doc[munged_key] = entry_doc.pop(key, None) + def sync_phone_to_server(uuid, data_from_phone): """ Puts the blob from the phone into the cache @@ -44,6 +57,10 @@ def sync_phone_to_server(uuid, data_from_phone): if "ts" in data["data"] and ecc.isMillisecs(data["data"]["ts"]): data["data"]["ts"] = old_div(float(data["data"]["ts"]), 1000) + + # mongodb/documentDB don't support field names with `.` + # let's convert them all to `_` + _remove_dots(data) # logging.debug("After updating with UUId, we get %s" % data) document = {'$set': data} diff --git a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py index 3d024be06..8c4c82ad7 100644 --- a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py +++ b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py @@ -14,6 +14,7 @@ import uuid import attrdict as ad import time +import copy import geojson as gj # This change should be removed in the next server update, by which time hopefully the new geojson version will incorporate the long-term fix for their default precision # See - jazzband/geojson#177 @@ -273,6 +274,77 @@ def testTwoLongTermCalls(self): self.assertEqual(edb.get_timeseries_db().estimated_document_count(), 120) self.assertEqual(edb.get_timeseries_error_db().estimated_document_count(), 0) + def testRemoteDots(self): + test_template = {"ts":1735934360.256, + "client_app_version":"1.9.6", + "name":"open_notification", + "client_os_version":"15.5", + "reading":{ + "additionalData":{ + "google.c.sender.id":"766801492494", + "coldstart":False, + "notId":"1735934357036293", + "payload":1735934357036293, + "content-available":1, + "foreground":False, + "google.c.fid":"cW3YWb11wds", + "gcm.message_id":"1735934360053131"}}} + test_1 = copy.copy(test_template) + self.assertEqual(len(test_1["reading"]["additionalData"]), 8) + self.assertIn("google.c.sender.id", + test_1["reading"]["additionalData"]) + self.assertIn("google.c.fid", + test_1["reading"]["additionalData"]) + self.assertIn("gcm.message_id", + test_1["reading"]["additionalData"]) + mauc._remove_dots(test_1) + self.assertEqual(len(test_1["reading"]["additionalData"]), 8) + self.assertIn("google_c_sender_id", + test_1["reading"]["additionalData"]) + self.assertIn("google_c_fid", + test_1["reading"]["additionalData"]) + self.assertIn("gcm_message_id", + test_1["reading"]["additionalData"]) + self.assertNotIn("google.c.sender.id", + test_1["reading"]["additionalData"]) + self.assertNotIn("google.c.fid", + test_1["reading"]["additionalData"]) + self.assertNotIn("gcm.message_id", + test_1["reading"]["additionalData"]) + + metadata_template = {'plugin': 'none', + 'write_ts': self.curr_ts - 25, + 'time_zone': u'America/Los_Angeles', + 'platform': u'ios', + 'key': u'stats/client_time', + 'read_ts': self.curr_ts - 27, + 'type': u'message'} + + # there are 30 entries in the setup function + self.assertEqual(len(self.uc1.getMessage()), 30) + + three_entries_with_dots = [] + for i in range(3): + curr_md = copy.copy(metadata_template) + curr_md['write_ts'] = self.curr_ts - 25 + i + three_entries_with_dots.append({ + 'user_id': self.testUserUUID1, + 'data': copy.copy(test_template), + 'metadata': curr_md}) + + print(f"AFTER {[e.get('metadata', None) for e in three_entries_with_dots]}") + + mauc.sync_phone_to_server(self.testUserUUID1, three_entries_with_dots) + # we have munged, so these new entries should also be saved + # and we should have 33 entries in the usercache + self.assertEqual(len(self.uc1.getMessage()), 33) + self.assertEqual(len(list(self.ts1.find_entries())), 0) + enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() + # since they were munged before saving into the usercache, + # there should be no errors while copying to the timeseries + self.assertEqual(len(self.uc1.getMessage()), 0) + self.assertEqual(len(list(self.ts1.find_entries())), 33) + if __name__ == '__main__': import emission.tests.common as etc