Merge pull request #6267 from tomachalek/stable_conc_pers_fix

Fixes and updates in stable_conc_persistence, improve generate_idempo…
czcorpus · Jul 24, 2024 · 2b6860c · 2b6860c
2 parents 5c9c1a1 + 8bd6ad3
commit 2b6860c
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 14 deletions.
diff --git a/lib/plugin_types/query_persistence/common.py b/lib/plugin_types/query_persistence/common.py
@@ -57,15 +57,17 @@ def generate_uniq_id(_) -> str:
     return _encode_to_az(uuid.uuid1().int)
 
 
-def generate_idempotent_hex_id(data: Dict[str, Any]) -> str:
-    tmp = _to_json(data)
-    if tmp:
-        return hashlib.md5(tmp.encode('utf-8')).hexdigest()
-    else:
-        return generate_uniq_id(data)
-
-
 def generate_idempotent_id(data: Dict[str, Any]) -> str:
+    """
+    Based on `data` contents, generate a checksum/hash representing the data.
+    This ensures that the same record will always have the same ID.
+
+    In case empty data is provided, a unique ID is returned.
+    """
+    data = data.copy()  # shallow copy is ok here
+    # We don't want any existing ID nor user ID to affect generated hash
+    data.pop(ID_KEY, None)
+    data.pop(USER_ID_KEY, None)
     tmp = _to_json(data)
     if tmp:
         return _encode_to_az(int('0x' + hashlib.md5(tmp.encode('utf-8')).hexdigest(), 16))

diff --git a/lib/plugins/stable_query_persistence/__init__.py b/lib/plugins/stable_query_persistence/__init__.py
@@ -29,9 +29,7 @@
 """
 
 import logging
-import os
 import re
-import sqlite3
 import time
 import asyncio
 import aiosqlite
@@ -43,7 +41,7 @@
 from plugin_types.auth import AbstractAuth
 from plugin_types.query_persistence import AbstractQueryPersistence
 from plugin_types.query_persistence.common import (
-    ID_KEY, QUERY_KEY, USER_ID_KEY, generate_idempotent_hex_id)
+    ID_KEY, QUERY_KEY, USER_ID_KEY, generate_idempotent_id)
 from plugins import inject
 
 DEFAULT_TTL_DAYS = 7
@@ -142,7 +140,7 @@ def records_differ(r1, r2):
         if prev_data is None or records_differ(curr_data, prev_data):
             if prev_data is not None:
                 curr_data['prev_id'] = prev_data[ID_KEY]
-            data_id = generate_idempotent_hex_id(curr_data)
+            data_id = generate_idempotent_id(curr_data)
             curr_data[ID_KEY] = data_id
             curr_data[USER_ID_KEY] = user_id
             data_key = mk_key(data_id)
@@ -154,7 +152,7 @@ def records_differ(r1, r2):
 
         return latest_id
 
-    async def archive(self, user_id, conc_id, revoke=False):
+    async def archive(self, user_id, conc_id):
         async with self._archive_lock:
             async with aiosqlite.connect(self._archive_db_path) as db:
                 data = await self.db.get(mk_key(conc_id))
@@ -168,7 +166,7 @@ async def archive(self, user_id, conc_id, revoke=False):
                 await db.execute(
                     'INSERT OR IGNORE INTO archive (id, data, created, num_access) VALUES (?, ?, ?, ?)',
                     (conc_id, json.dumps(data), curr_time, 0))
-            await db.commit()
+                await db.commit()
             return data