New database cannot be atomic

.devcontainer/devcontainer.json

@@ -68,6 +68,13 @@
 		"ghcr.io/wxw-matt/devcontainer-features/command_runner:0": {},
 		"ghcr.io/audacioustux/devcontainers/common-utils-extras:1": {},
 		"ghcr.io/prulloac/devcontainer-features/gitlint:1": {}
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"GitHub.copilot"
+			]
+		}
 	}
 
 	// Features to add to the dev container. More info: https://containers.dev/features.

src/settings.py

@@ -13,7 +13,7 @@
 TASK_NAMES = ["IHM", "DECOMP", "LOS", "PHENO", "MULTI"]
 TEXT_METRICS = ["classification_report", "confusion_matrix"]
 
-with Path(os.getenv("CONFIG"), "datasets.json").open() as file:
+with Path(os.getenv("ETC"), "datasets.json").open() as file:
     DATASET_SETTINGS = json.load(file)
     DECOMP_SETTINGS = DATASET_SETTINGS["DECOMP"]
     LOS_SETTINGS = DATASET_SETTINGS["LOS"]

tests/test_datasets/test_trackers/test_storage_decorator.py

@@ -1,9 +1,10 @@
+from tests.tsettings import *
 import pytest
 import shutil
 from pathlib import Path
 from utils.IO import *
-from tests.tsettings import *
-from storable import storable
+# from storable import storable
+from trash1 import storable
 
 
 @storable
@@ -279,21 +280,21 @@ def test_list_basic():
 
     test_instance.subject_ids = ["a", "b", "c"]
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     tests_io("Succeeded testing basic list assignment.")
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
     tests_io("Succeeded testing restoration of basic list assignment.")
     # We are tempering with the cls so need to make sure nothing is permanent
     del test_instance
     if TEMP_DIR.is_dir():
         shutil.rmtree(str(TEMP_DIR))
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == []
-    assert test_instance._progress["subject_ids"] == []
+    # assert test_instance._progress["subject_ids"] == []
     tests_io("Succeeded testing list non permanency on database deletion.")
 
 
@@ -303,24 +304,24 @@ def test_list_extend():
 
     test_instance.subject_ids.extend(["a", "b", "c"])
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
     tests_io("Succeeded testing extend list assignment.")
 
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
     tests_io("Succeeded testing restoration of extend list assignment.")
 
     test_instance.subject_ids.extend(["d", "e", "f"])
     assert test_instance.subject_ids == ["a", "b", "c", "d", "e", "f"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d", "e", "f"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d", "e", "f"]
     tests_io("Succeeded testing extend list assignment.")
 
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c", "d", "e", "f"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d", "e", "f"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d", "e", "f"]
     tests_io("Succeeded testing restoration of extend list assignment.")
 
 
@@ -331,25 +332,25 @@ def test_list_pop():
     # Assign and make sure
     test_instance.subject_ids = ["a", "b", "c"]
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Now pop
     popper = test_instance.subject_ids.pop()
     assert popper == "c"
     assert test_instance.subject_ids == ["a", "b"]
-    assert test_instance._progress["subject_ids"] == ["a", "b"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b"]
     tests_io("Succeeded testing pop list assignment.")
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b"]
-    assert test_instance._progress["subject_ids"] == ["a", "b"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b"]
     tests_io("Succeeded testing restoration of pop list assignment.")
 
 
@@ -360,25 +361,25 @@ def test_list_remove():
     # Assign and make sure
     test_instance.subject_ids = ["a", "b", "c"]
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Now remove
     test_instance.subject_ids.remove("b")
     assert test_instance.subject_ids == ["a", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "c"]
     tests_io("Succeeded testing remove list assignment.")
 
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "c"]
     tests_io("Succeeded testing restoration of remove list assignment.")
 
 
@@ -389,28 +390,90 @@ def test_list_append():
     # Assign and make sure
     test_instance.subject_ids = ["a", "b", "c"]
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c"]
 
     # Now append
     test_instance.subject_ids.append("d")
     assert test_instance.subject_ids == ["a", "b", "c", "d"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d"]
     tests_io("Succeeded testing append list assignment.")
 
     # Restore to see if permament
     del test_instance
     test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
     assert test_instance.subject_ids == ["a", "b", "c", "d"]
-    assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d"]
+    # assert test_instance._progress["subject_ids"] == ["a", "b", "c", "d"]
     tests_io("Succeeded testing restoration of append list assignment.")
 
 
+def update_instance(instance, iterations, thread_id):
+    print(f"Thread {thread_id} starting. Initial state: {instance.__dict__}")
+    for i in range(iterations):
+        instance.num_samples += 1
+        instance.time_elapsed += 0.1
+        instance.subject_ids.append(random.randint(1, 1000))
+        key = random.choice(['a', 'b'])
+        instance.subjects[key] += 1
+    instance.finished = True
+
+
+import pytest
+from concurrent.futures import ThreadPoolExecutor
+import time
+import random
+
+
+def test_concurrent_access():
+    test_instance = CountTestClass(Path(TEMP_DIR, "progress"))
+
+    num_threads = 5
+    iterations_per_thread = 100
+
+    with ThreadPoolExecutor(max_workers=num_threads) as executor:
+        futures = [
+            executor.submit(update_instance, test_instance, iterations_per_thread, i)
+            for i in range(num_threads)
+        ]
+
+        # Wait for all threads to complete
+        for future in futures:
+            future.result()
+
+    print(f"All threads completed. Final state: {test_instance.__dict__}")
+
+    # Verify the results
+    assert test_instance.num_samples == num_threads * iterations_per_thread, f"Expected {num_threads * iterations_per_thread}, got {test_instance.num_samples}"
+    assert pytest.approx(test_instance.time_elapsed,
+                         abs=0.1) == num_threads * iterations_per_thread * 0.1
+    assert len(
+        test_instance.subject_ids
+    ) == num_threads * iterations_per_thread, f"Expected {num_threads * iterations_per_thread}, got {len(test_instance.subject_ids)}"
+    assert test_instance.subjects[
+        "total"] == num_threads * iterations_per_thread, f"Expected {num_threads * iterations_per_thread}, got {sum(test_instance.subjects.values())}"
+    assert test_instance.finished
+
+
+def test_persistence(test_instance):
+    print(f"Persistence test starting. Original instance state: {test_instance.__dict__}")
+    # Verify persistence by creating a new instance
+    new_instance = CountTestClass()
+    print(f"New instance created for persistence test. State: {new_instance.__dict__}")
+
+    assert new_instance.num_samples == test_instance.num_samples, f"num_samples mismatch: {new_instance.num_samples} != {test_instance.num_samples}"
+    assert pytest.approx(
+        new_instance.time_elapsed, abs=0.1
+    ) == test_instance.time_elapsed, f"time_elapsed mismatch: {new_instance.time_elapsed} != {test_instance.time_elapsed}"
+    assert new_instance.subject_ids == test_instance.subject_ids, f"subject_ids mismatch: {new_instance.subject_ids} != {test_instance.subject_ids}"
+    assert new_instance.subjects == test_instance.subjects, f"subjects mismatch: {new_instance.subjects} != {test_instance.subjects}"
+    assert new_instance.finished == test_instance.finished, f"finished mismatch: {new_instance.finished} != {test_instance.finished}"
+
+
 def check_dtypes(instance):
     assert isinstance(instance.num_samples, int)
     assert isinstance(instance.time_elapsed, float)
@@ -420,6 +483,9 @@ def check_dtypes(instance):
 
 
 if __name__ == "__main__":
+    # if TEMP_DIR.is_dir():
+    #     shutil.rmtree(str(TEMP_DIR))
+    # test_concurrent_access()
     if TEMP_DIR.is_dir():
         shutil.rmtree(str(TEMP_DIR))
     test_list_basic()