diff --git a/api/aws_lambdas/scorer_api_passport/v1/score_POST.py b/api/aws_lambdas/scorer_api_passport/v1/score_POST.py index 88a485d15..6c5ce27cc 100644 --- a/api/aws_lambdas/scorer_api_passport/v1/score_POST.py +++ b/api/aws_lambdas/scorer_api_passport/v1/score_POST.py @@ -5,10 +5,14 @@ from aws_lambdas.scorer_api_passport.utils import ( authenticate_and_get_address, format_response, - with_request_exception_handling, parse_body, + with_request_exception_handling, ) from ceramic_cache.api.v1 import get_detailed_score_response_for_address + +DUMMY_MARKER = "import django.db stuff after this line" +# django.db needs to be imported after the aws helpers +from django.conf import settings from django.db import close_old_connections @@ -16,7 +20,9 @@ def _handler(event, context): address = authenticate_and_get_address(event) body = parse_body(event) - alternate_scorer_id = body.get("alternate_scorer_id", None) + alternate_scorer_id = ( + body.get("alternate_scorer_id", None) or settings.CERAMIC_CACHE_SCORER_ID + ) return format_response( get_detailed_score_response_for_address(address, alternate_scorer_id) diff --git a/api/aws_lambdas/scorer_api_passport/v1/stamps/bulk_POST.py b/api/aws_lambdas/scorer_api_passport/v1/stamps/bulk_POST.py index 5ad2ea287..1ea053b56 100644 --- a/api/aws_lambdas/scorer_api_passport/v1/stamps/bulk_POST.py +++ b/api/aws_lambdas/scorer_api_passport/v1/stamps/bulk_POST.py @@ -8,9 +8,15 @@ parse_body, with_request_exception_handling, ) -from ceramic_cache.api.v1 import CacheStampPayload, handle_add_stamps + +""" +Imports after aws_lambdas.scorer_api_passport.utils +""" from django.db import close_old_connections +from ceramic_cache.api.v1 import CacheStampPayload, handle_add_stamps +from ceramic_cache.models import CeramicCache + @with_request_exception_handling def _handler(event, context): @@ -19,7 +25,9 @@ def _handler(event, context): payload = [CacheStampPayload(**p) for p in body] - return format_response(handle_add_stamps(address, payload)) + return format_response( + handle_add_stamps(address, payload, CeramicCache.SourceApp.PASSPORT) + ) def handler(*args, **kwargs): diff --git a/api/ceramic_cache/admin.py b/api/ceramic_cache/admin.py index d858d57ea..91a9bc5ae 100644 --- a/api/ceramic_cache/admin.py +++ b/api/ceramic_cache/admin.py @@ -56,13 +56,15 @@ class CeramicCacheAdmin(ScorerModelAdmin): "id", "address", "provider", - "stamp", "deleted_at", "compose_db_save_status", "compose_db_stream_id", "proof_value", + "source_app", + "source_scorer_id", + "stamp", ) - list_filter = ("deleted_at", "compose_db_save_status") + list_filter = ("deleted_at", "compose_db_save_status", "source_app") search_fields = ("address__exact", "compose_db_stream_id__exact", "proof_value") search_help_text = ( "This will perform a search by 'address' and 'compose_db_stream_id'" diff --git a/api/ceramic_cache/api/v1.py b/api/ceramic_cache/api/v1.py index cb0cbfcab..b707f500a 100644 --- a/api/ceramic_cache/api/v1.py +++ b/api/ceramic_cache/api/v1.py @@ -7,7 +7,6 @@ from typing import Any, Dict, List, Optional, Type import requests -from asgiref.sync import async_to_sync from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import AbstractUser @@ -36,7 +35,7 @@ DetailedScoreResponse, ErrorMessageResponse, SubmitPassportPayload, - ahandle_submit_passport, + handle_submit_passport, ) from registry.exceptions import ( InvalidAddressException, @@ -139,14 +138,22 @@ def cache_stamps(request, payload: List[CacheStampPayload]): try: address = get_address_from_did(request.did) - return handle_add_stamps(address, payload) + return handle_add_stamps( + address, + payload, + CeramicCache.SourceApp.PASSPORT, + settings.CERAMIC_CACHE_SCORER_ID, + ) except Exception as e: raise e def handle_add_stamps_only( - address, payload: List[CacheStampPayload], alternate_scorer_id: Optional[int] = None + address, + payload: List[CacheStampPayload], + source_app: CeramicCache.SourceApp, + alternate_scorer_id: Optional[int] = None, ) -> GetStampResponse: if len(payload) > settings.MAX_BULK_CACHE_SIZE: raise TooManyStampsException() @@ -173,6 +180,8 @@ def handle_add_stamps_only( compose_db_save_status=CeramicCache.ComposeDBSaveStatus.PENDING, issuance_date=p.stamp.get("issuanceDate", None), expiration_date=p.stamp.get("expirationDate", None), + source_app=source_app, + source_scorer_id=alternate_scorer_id, ) for p in payload ] @@ -201,16 +210,19 @@ def handle_add_stamps_only( def handle_add_stamps( - address, payload: List[CacheStampPayload], alternate_scorer_id: Optional[int] = None + address, + payload: List[CacheStampPayload], + stamp_creator: CeramicCache.SourceApp, + alternate_scorer_id: Optional[int] = None, ) -> GetStampsWithScoreResponse: - - stamps_response = handle_add_stamps_only(address, payload, alternate_scorer_id) + stamps_response = handle_add_stamps_only( + address, payload, stamp_creator, alternate_scorer_id + ) + scorer_id = alternate_scorer_id or settings.CERAMIC_CACHE_SCORER_ID return GetStampsWithScoreResponse( success=stamps_response.success, stamps=stamps_response.stamps, - score=get_detailed_score_response_for_address( - address, alternate_scorer_id=alternate_scorer_id - ), + score=get_detailed_score_response_for_address(address, scorer_id=scorer_id), ) @@ -222,13 +234,13 @@ def patch_stamps(request, payload: List[CacheStampPayload]): address = get_address_from_did(request.did) return handle_patch_stamps(address, payload) - except Exception: + except Exception as exc: log.error( "Failed patch_stamps request: '%s'", - [p.dict() for p in payload], + [p.model_dump_json() for p in payload], exc_info=True, ) - raise InternalServerException() + raise InternalServerException() from exc def handle_patch_stamps( @@ -287,7 +299,9 @@ def handle_patch_stamps( ) for stamp in updated_passport_state ], - score=get_detailed_score_response_for_address(address), + score=get_detailed_score_response_for_address( + address, settings.CERAMIC_CACHE_SCORER_ID + ), ) @@ -391,7 +405,9 @@ def handle_delete_stamps( ) for stamp in updated_passport_state ], - score=get_detailed_score_response_for_address(address), + score=get_detailed_score_response_for_address( + address, settings.CERAMIC_CACHE_SCORER_ID + ), ) @@ -454,7 +470,7 @@ def handle_get_stamps(address): passport__community_id=scorer_id, ).exists() ): - get_detailed_score_response_for_address(address) + get_detailed_score_response_for_address(address, scorer_id) return GetStampResponse( success=True, @@ -650,9 +666,8 @@ def handle_authenticate(payload: CacaoVerifySubmit) -> AccessTokenResponse: def get_detailed_score_response_for_address( - address: str, alternate_scorer_id: Optional[int] = None + address: str, scorer_id: Optional[int] ) -> DetailedScoreResponse: - scorer_id = alternate_scorer_id or settings.CERAMIC_CACHE_SCORER_ID if not scorer_id: raise InternalServerException("Scorer ID not set") @@ -663,8 +678,7 @@ def get_detailed_score_response_for_address( scorer_id=str(scorer_id), ) - score = async_to_sync(ahandle_submit_passport)(submit_passport_payload, account) - + score = handle_submit_passport(submit_passport_payload, account) return score diff --git a/api/ceramic_cache/migrations/0031_ceramiccache_source_app_and_more.py b/api/ceramic_cache/migrations/0031_ceramiccache_source_app_and_more.py new file mode 100644 index 000000000..907266ca8 --- /dev/null +++ b/api/ceramic_cache/migrations/0031_ceramiccache_source_app_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.6 on 2025-01-14 16:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ceramic_cache", "0030_alter_ceramiccache_created_at_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="ceramiccache", + name="source_app", + field=models.IntegerField( + blank=True, + choices=[(1, "Passport"), (2, "Embed")], + db_index=True, + help_text="Which entity created the stamp. At the moment there are 2 options: the 'Passport App' and 'Embed Widget'", + null=True, + verbose_name="Creating Enity", + ), + ), + migrations.AddField( + model_name="ceramiccache", + name="source_scorer_id", + field=models.BigIntegerField( + blank=True, + db_index=True, + help_text="This is field is only used to indicate for analytic purposes which scorer was targeted \n when claiming the users credential (when used from embed, it will indicate what scorer id was set in \n the embed component)", + null=True, + verbose_name="Scorer ID", + ), + ), + ] diff --git a/api/ceramic_cache/models.py b/api/ceramic_cache/models.py index 8721bc09e..b7fa8882c 100644 --- a/api/ceramic_cache/models.py +++ b/api/ceramic_cache/models.py @@ -16,6 +16,10 @@ class StampType(IntEnum): V1 = 1 V2 = 2 + class SourceApp(models.IntegerChoices): + PASSPORT = 1 + EMBED = 2 + class ComposeDBSaveStatus(models.TextChoices): PENDING = "pending" SAVED = "saved" @@ -84,6 +88,30 @@ class ComposeDBSaveStatus(models.TextChoices): null=True, db_index=True ) # stamp['expirationDate'] + ################################################################################################ + # Begin metadata fields, to determine the scorer that initiated the creation of a stamp + ################################################################################################# + source_scorer_id = models.BigIntegerField( + verbose_name="Scorer ID", + help_text="""This is field is only used to indicate for analytic purposes which scorer was targeted + when claiming the users credential (when used from embed, it will indicate what scorer id was set in + the embed component)""", + null=True, + blank=True, + db_index=True, + ) + source_app = models.IntegerField( + verbose_name="Creating Enity", + help_text="""Which entity created the stamp. At the moment there are 2 options: the 'Passport App' and 'Embed Widget'""", + choices=SourceApp.choices, + null=True, + blank=True, + db_index=True, + ) + ################################################################################################ + # End metadata fields + ################################################################################################# + class Meta: unique_together = ["type", "address", "provider", "deleted_at"] diff --git a/api/ceramic_cache/test/conftest.py b/api/ceramic_cache/test/conftest.py index a0049a6e6..7a81c3d48 100644 --- a/api/ceramic_cache/test/conftest.py +++ b/api/ceramic_cache/test/conftest.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta, timezone + import pytest from django.conf import settings @@ -28,11 +30,29 @@ def sample_providers(): @pytest.fixture -def sample_stamps(): +def sample_expiration_dates(sample_providers): + now = datetime.now(timezone.utc) + return [now + timedelta(days=idx) for idx, _ in enumerate(sample_providers, 1)] + + +@pytest.fixture +def sample_stamps(sample_expiration_dates, sample_providers, sample_address): return [ - {"stamp": 1, "proof": {"proofValue": "test1"}}, - {"stamp": 2, "proof": {"proofValue": "test2"}}, - {"stamp": 3, "proof": {"proofValue": "test3"}}, + { + "type": ["VerifiableCredential"], + "credentialSubject": { + "id": sample_address, + "hash": "v0.0.0:1Vzw/OyM9CBUkVi/3mb+BiwFnHzsSRZhVH1gaQIyHvM=", + "provider": sample_providers[idx], + }, + "issuer": settings.TRUSTED_IAM_ISSUERS[0], + "issuanceDate": (expiration_date - timedelta(days=30)).isoformat(), + "expirationDate": expiration_date.isoformat(), + "proof": { + "proofValue": "proof-v0.0.0:1Vzw/OyM9CBUkVi/3mb+BiwFnHzsSRZhVH1gaQIyHvM=", + }, + } + for idx, expiration_date in enumerate(sample_expiration_dates) ] diff --git a/api/ceramic_cache/test/test_bulk_updates_compose_db.py b/api/ceramic_cache/test/test_bulk_updates_compose_db.py new file mode 100644 index 000000000..95e27a104 --- /dev/null +++ b/api/ceramic_cache/test/test_bulk_updates_compose_db.py @@ -0,0 +1,95 @@ +import json + +import pytest + +from ceramic_cache.models import CeramicCache + +pytestmark = pytest.mark.django_db + + +class TestComposeDBBulkStampUpdates: + base_url = "/ceramic-cache" + stamp_version = CeramicCache.StampType.V1 + + def test_success_bulk_update_compose_db_status( + self, + sample_providers, + sample_address, + sample_stamps, + sample_token, + ui_scorer, + client, + ): + # Create stamps first + create_response = client.post( + f"{self.base_url}/stamps/bulk", + json.dumps( + [ + { + "provider": sample_providers[0], + "stamp": sample_stamps[0], + }, + { + "provider": sample_providers[1], + "stamp": sample_stamps[1], + }, + # This one will stay pending + { + "provider": sample_providers[2], + "stamp": sample_stamps[2], + }, + ] + ), + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, + ) + assert create_response.status_code == 201 + + ids = [stamp["id"] for stamp in create_response.json()["stamps"]] + + missing_id = 1234564564564564564 + + # Construct payload + bulk_payload = [ + { + "id": ids[0], + "compose_db_save_status": "saved", + "compose_db_stream_id": "stream-id-1", + }, + { + "id": ids[1], + "compose_db_save_status": "failed", + }, + # This is not valid, but should not interfere + { + "id": missing_id, + "compose_db_save_status": "saved", + }, + ] + + # Make the request + response = client.patch( + f"{self.base_url}/stamps/bulk/meta/compose-db", + json.dumps(bulk_payload), + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, + ) + + assert response.status_code == 200 + + data = response.json() + + assert len(data["updated"]) == 2 + + assert missing_id not in data["updated"] + assert ids[0] in data["updated"] + assert ids[1] in data["updated"] + + assert CeramicCache.objects.filter(compose_db_save_status="saved").count() == 1 + assert CeramicCache.objects.filter(compose_db_save_status="failed").count() == 1 + assert ( + CeramicCache.objects.filter(compose_db_save_status="pending").count() == 1 + ) + assert ( + CeramicCache.objects.filter(compose_db_stream_id="stream-id-1").count() == 1 + ) diff --git a/api/ceramic_cache/test/test_bulk_updates_v1.py b/api/ceramic_cache/test/test_bulk_updates_v1.py index 0a8426408..1f903d154 100644 --- a/api/ceramic_cache/test/test_bulk_updates_v1.py +++ b/api/ceramic_cache/test/test_bulk_updates_v1.py @@ -1,21 +1,28 @@ import json +from copy import deepcopy from datetime import datetime +from unittest.mock import patch import pytest -from django.test import Client from ceramic_cache.api.v1 import get_address_from_did from ceramic_cache.models import CeramicCache pytestmark = pytest.mark.django_db -client = Client() + +def ascore_passport_patch(_community, _passport, _address, score): + score.status = "DONE" class TestBulkStampUpdates: base_url = "/ceramic-cache" stamp_version = CeramicCache.StampType.V1 + @patch( + "registry.api.v1.ascore_passport", + side_effect=ascore_passport_patch, + ) def test_bulk_create( self, sample_providers, @@ -23,6 +30,7 @@ def test_bulk_create( sample_stamps, sample_token, ui_scorer, + client, ): bulk_payload = [] for i in range(0, len(sample_providers)): @@ -41,18 +49,81 @@ def test_bulk_create( ) assert cache_stamp_response.status_code == 201 - stamps = cache_stamp_response.json()["stamps"] + cache_stamp_response_data = cache_stamp_response.json() + stamps = cache_stamp_response_data["stamps"] + sorted_stamps = sorted(stamps, key=lambda x: x["stamp"]["issuanceDate"]) + score = cache_stamp_response_data["score"] assert len(stamps) == len(sample_providers) - assert stamps[0]["id"] is not None + sorted_sample_stamps = sorted(sample_stamps, key=lambda x: x["issuanceDate"]) + + # Verify the returned score + assert score == { + "address": sample_address.lower(), + "error": None, + "evidence": None, + "expiration_date": None, + "last_score_timestamp": None, + "score": None, + "stamp_scores": {}, + "status": "DONE", + } + + # Verify the returned stamps + for i in range(0, len(sorted_stamps)): + provider = sorted_sample_stamps[i]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps[i] + assert sorted_stamps[i] == { + # For these fields, values are set by the server + "id": sorted_stamps[i]["id"], + # For the attributes below we know the expected values + "address": sample_address.lower(), + "provider": provider, + "stamp": stamp, + } + + # Test that stamps are stored correctly + cc = sorted( + CeramicCache.objects.all().values(), + key=lambda x: x["stamp"]["issuanceDate"], + ) + for idx, c in enumerate(cc): + provider = sorted_sample_stamps[idx]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps[idx] + assert c == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + # Here are the values we control + "address": sample_address.lower(), + "compose_db_save_status": "pending", + "compose_db_stream_id": "", + "deleted_at": None, + "expiration_date": datetime.fromisoformat(stamp["expirationDate"]), + "issuance_date": datetime.fromisoformat(stamp["issuanceDate"]), + "proof_value": stamp["proof"]["proofValue"], + "provider": provider, + "scorer_id": ui_scorer, + "stamp": stamp, + "source_app": CeramicCache.SourceApp.PASSPORT.value, + "type": 1, + } + + @patch( + "registry.api.v1.ascore_passport", + side_effect=ascore_passport_patch, + ) def test_bulk_update( self, + _avalidate_credentials, sample_providers, sample_address, sample_stamps, sample_token, scorer_account, ui_scorer, + client, ): bulk_payload = [] for i in range(0, len(sample_providers)): @@ -71,17 +142,17 @@ def test_bulk_update( ) assert cache_stamp_response.status_code == 201 - assert len(cache_stamp_response.json()["stamps"]) == len(sample_providers) bulk_payload = [] - for i in range(0, len(sample_providers)): - bulk_payload.append( - { - "provider": sample_providers[i], - "stamp": {"updated": True, "proof": {"proofValue": "updated"}}, - } - ) + sorted_sample_stamps = sorted(sample_stamps, key=lambda x: x["issuanceDate"]) + sorted_sample_stamps_updates = [] + + for i, stamp in enumerate(sorted_sample_stamps): + new_stamp = deepcopy(stamp) + new_stamp["proof"]["proofValue"] = f"updated {i}" + sorted_sample_stamps_updates.append(new_stamp) + bulk_payload.append({"provider": sample_providers[i], "stamp": new_stamp}) cache_stamp_response = client.post( f"{self.base_url}/stamps/bulk", @@ -92,24 +163,90 @@ def test_bulk_update( assert cache_stamp_response.status_code == 201 - stamps = cache_stamp_response.json()["stamps"] + response = cache_stamp_response.json() + stamps = response["stamps"] + score = response["score"] assert len(stamps) == len(sample_providers) - for i in range(0, len(sample_providers)): - assert stamps[i]["stamp"]["updated"] == True - assert stamps[i]["id"] is not None - def test_bulk_patch( + assert score == { + "address": sample_address.lower(), + "error": None, + "evidence": None, + "expiration_date": None, + "last_score_timestamp": None, + "score": None, + "stamp_scores": {}, + "status": "DONE", + } + + sorted_stamps_returned = sorted( + stamps, key=lambda x: x["stamp"]["issuanceDate"] + ) + + for i in range(0, len(sorted_sample_stamps_updates)): + provider = sorted_sample_stamps_updates[i]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps_updates[i] + assert sorted_stamps_returned[i] == { + # For these fields, values are set by the server + "id": sorted_stamps_returned[i]["id"], + # For the attributes below we know the expected values + "address": sample_address.lower(), + "provider": provider, + "stamp": stamp, + } + + # Test that stamps are stored correctly + cc = sorted( + list(CeramicCache.objects.filter(deleted_at__isnull=True).values()), + key=lambda x: x["stamp"]["issuanceDate"], + ) + + for i, c in enumerate(cc): + provider = sorted_sample_stamps_updates[i]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps_updates[i] + assert c == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + # Here are the values we control + "address": sample_address.lower(), + "compose_db_save_status": "pending", + "compose_db_stream_id": "", + "deleted_at": None, + "expiration_date": datetime.fromisoformat(stamp["expirationDate"]), + "issuance_date": datetime.fromisoformat(stamp["issuanceDate"]), + "proof_value": stamp["proof"]["proofValue"], + "provider": provider, + "source_scorer_id": ui_scorer, + "stamp": stamp, + "source_app": CeramicCache.SourceApp.PASSPORT.value, + "type": 1, + } + + @patch( + "registry.api.v1.ascore_passport", + side_effect=ascore_passport_patch, + ) + def test_bulk_patch_partial_update( self, + _avalidate_credentials, sample_providers, sample_address, sample_stamps, sample_token, + scorer_account, ui_scorer, + client, ): - # create two stamps + """ + Test updating only a part of the stamps + """ + assert CeramicCache.objects.all().count() == 0 + bulk_payload = [] - for i in range(0, 2): + for i in range(0, len(sample_providers)): bulk_payload.append( { "provider": sample_providers[i], @@ -117,83 +254,138 @@ def test_bulk_patch( } ) - cache_stamp_response = client.patch( + cache_stamp_response = client.post( f"{self.base_url}/stamps/bulk", json.dumps(bulk_payload), content_type="application/json", **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, ) - assert cache_stamp_response.status_code == 200 - stamps = cache_stamp_response.json()["stamps"] - assert len(stamps) == 2 - assert stamps[0]["id"] is not None - - # Should have a stamp for the first provider, but not for last provider - assert ( - CeramicCache.objects.filter( - type=self.stamp_version, provider=sample_providers[0] - ).count() - == 1 - ) - - assert ( - CeramicCache.objects.filter( - type=self.stamp_version, - provider=sample_providers[len(sample_providers) - 1], - ).count() - == 0 - ) + assert cache_stamp_response.status_code == 201 + assert len(cache_stamp_response.json()["stamps"]) == len(sample_providers) - # patch all the stamps except the first, which is deleted - bulk_payload = [{"provider": sample_providers[0]}] - for i in range(1, len(sample_providers)): - bulk_payload.append( - { - "provider": sample_providers[i], - "stamp": {"updated": True, "proof": {"proofValue": "test"}}, - } - ) + bulk_payload = [] + sorted_sample_stamps = sorted(sample_stamps, key=lambda x: x["issuanceDate"]) + # This will hold the state we expect when getting the stamp list back (or reading from DB) + sorted_sample_stamps_updated = [] + + for i, stamp in enumerate(sorted_sample_stamps): + if i < len(sample_providers) / 2: + new_stamp = deepcopy(stamp) + new_stamp["proof"]["proofValue"] = f"updated {i}" + sorted_sample_stamps_updated.append(new_stamp) + bulk_payload.append( + {"provider": sample_providers[i], "stamp": new_stamp} + ) + else: + sorted_sample_stamps_updated.append(deepcopy(stamp)) - cache_stamp_response = client.patch( + cache_stamp_response = client.post( f"{self.base_url}/stamps/bulk", json.dumps(bulk_payload), content_type="application/json", **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, ) - assert cache_stamp_response.status_code == 200 - assert len(cache_stamp_response.json()["stamps"]) == len(sample_providers) - 1 + assert cache_stamp_response.status_code == 201 - # Should now have a stamp for the last provider, but stamp for first provider should be marked as deleted - assert ( - CeramicCache.objects.filter( - type=self.stamp_version, - provider=sample_providers[0], - deleted_at__isnull=True, - ).count() - == 0 + response = cache_stamp_response.json() + stamps = response["stamps"] + score = response["score"] + + assert len(stamps) == len(sample_providers) + + assert score == { + "address": sample_address.lower(), + "error": None, + "evidence": None, + "expiration_date": None, + "last_score_timestamp": None, + "score": None, + "stamp_scores": {}, + "status": "DONE", + } + + sorted_stamps_returned = sorted( + stamps, key=lambda x: x["stamp"]["issuanceDate"] ) - assert ( - CeramicCache.objects.filter( - type=self.stamp_version, - provider=sample_providers[0], - deleted_at__isnull=False, - ).count() - == 1 + for i in range(0, len(sorted_sample_stamps_updated)): + provider = sorted_sample_stamps_updated[i]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps_updated[i] + assert sorted_stamps_returned[i] == { + # For these fields, values are set by the server + "id": sorted_stamps_returned[i]["id"], + # For the attributes below we know the expected values + "address": sample_address.lower(), + "provider": provider, + "stamp": stamp, + } + + # Test that stamps are stored correctly + cc = sorted( + list(CeramicCache.objects.filter(deleted_at__isnull=True).values()), + key=lambda x: x["stamp"]["issuanceDate"], ) - assert ( - CeramicCache.objects.filter( - type=self.stamp_version, - provider=sample_providers[len(sample_providers) - 1], - ).count() - == 1 + for i, c in enumerate(cc): + provider = sorted_sample_stamps_updated[i]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps_updated[i] + assert c == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + # Here are the values we control + "address": sample_address.lower(), + "compose_db_save_status": "pending", + "compose_db_stream_id": "", + "deleted_at": None, + "expiration_date": datetime.fromisoformat(stamp["expirationDate"]), + "issuance_date": datetime.fromisoformat(stamp["issuanceDate"]), + "proof_value": stamp["proof"]["proofValue"], + "provider": provider, + "source_scorer_id": ui_scorer, + "stamp": stamp, + "source_app": CeramicCache.SourceApp.PASSPORT.value, + "type": 1, + } + + # Check that old versions of stamps are marked as deleted + cc = sorted( + list(CeramicCache.objects.filter(deleted_at__isnull=False).values()), + key=lambda x: x["stamp"]["issuanceDate"], ) + assert len(cc) == len(bulk_payload) + + for i, c in enumerate(cc): + provider = bulk_payload[i]["stamp"]["credentialSubject"]["provider"] + stamp = sorted_sample_stamps[i] + + assert c["deleted_at"] is not None + assert c == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + "deleted_at": c["deleted_at"], + # Here are the values we control + "address": sample_address.lower(), + "compose_db_save_status": "pending", + "compose_db_stream_id": "", + "expiration_date": datetime.fromisoformat(stamp["expirationDate"]), + "issuance_date": datetime.fromisoformat(stamp["issuanceDate"]), + "proof_value": stamp["proof"]["proofValue"], + "provider": provider, + "source_scorer_id": ui_scorer, + "stamp": stamp, + "source_app": CeramicCache.SourceApp.PASSPORT.value, + "type": 1, + } + def test_bulk_update_create_doesnt_accept_greater_than_100_stamps( - self, sample_providers, sample_address, sample_stamps, sample_token + self, sample_providers, sample_address, sample_stamps, sample_token, client ): bulk_payload = [] for i in range(0, 101): @@ -217,18 +409,34 @@ def test_bulk_update_create_doesnt_accept_greater_than_100_stamps( } def test_successful_bulk_delete( - self, sample_providers, sample_address, sample_stamps, sample_token, ui_scorer + self, + sample_providers, + sample_address, + sample_stamps, + sample_token, + ui_scorer, + client, ): + sorted_sample_stamps = sorted(sample_stamps, key=lambda x: x["issuanceDate"]) bulk_payload = [] - for i in range(0, 3): + for stamp in sorted_sample_stamps: CeramicCache.objects.create( type=self.stamp_version, address=sample_address, - provider=sample_providers[i], - stamp=sample_stamps[i], + provider=stamp["credentialSubject"]["provider"], + stamp=stamp, + source_app=CeramicCache.SourceApp.PASSPORT.value, + source_scorer_id=ui_scorer, + compose_db_save_status="pending", + expiration_date=datetime.fromisoformat(stamp["expirationDate"]), + issuance_date=datetime.fromisoformat(stamp["issuanceDate"]), + proof_value=stamp["proof"]["proofValue"], ) bulk_payload.append( - {"address": sample_address, "provider": sample_providers[i]} + { + "address": sample_address, + "provider": stamp["credentialSubject"]["provider"], + } ) cache_stamp_response = client.delete( @@ -239,8 +447,57 @@ def test_successful_bulk_delete( ) assert cache_stamp_response.status_code == 200 - assert cache_stamp_response.json()["success"] == True - assert len(cache_stamp_response.json()["stamps"]) == 0 + cache_stamp_response_data = cache_stamp_response.json() + assert cache_stamp_response_data["success"] is True + assert len(cache_stamp_response_data["stamps"]) == 0 + + # TODO: ideally score would be returned as well on DELETE, this + # should be fixed in the future + # score = cache_stamp_response_data["score"] + + # assert score == { + # "address": sample_address.lower(), + # "error": None, + # "evidence": None, + # "expiration_date": None, + # "last_score_timestamp": None, + # "score": None, + # "stamp_scores": {}, + # "status": "DONE", + # } + + # Check that old versions of stamps are marked as deleted + cc = sorted( + list(CeramicCache.objects.filter(deleted_at__isnull=False).values()), + key=lambda x: x["stamp"]["issuanceDate"], + ) + + assert len(cc) == len(sorted_sample_stamps) + + for i, c in enumerate(cc): + provider = bulk_payload[i]["provider"] + stamp = sorted_sample_stamps[i] + + assert c["deleted_at"] is not None + assert c == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + "deleted_at": c["deleted_at"], + # Here are the values we control + "address": sample_address.lower(), + "compose_db_save_status": "pending", + "compose_db_stream_id": "", + "expiration_date": datetime.fromisoformat(stamp["expirationDate"]), + "issuance_date": datetime.fromisoformat(stamp["issuanceDate"]), + "proof_value": stamp["proof"]["proofValue"], + "provider": provider, + "source_scorer_id": ui_scorer, + "stamp": stamp, + "source_app": CeramicCache.SourceApp.PASSPORT.value, + "type": 1, + } def test_bulk_delete_indicates_a_subset_of_stamps_were_deleted( self, @@ -249,6 +506,7 @@ def test_bulk_delete_indicates_a_subset_of_stamps_were_deleted( sample_stamps, sample_token, ui_scorer, + client, ): bulk_payload = [] for i in range(0, 3): @@ -275,7 +533,7 @@ def test_bulk_delete_indicates_a_subset_of_stamps_were_deleted( assert cache_stamp_response.json()["success"] == True def test_bulk_delete_indicates_no_stamps_were_deleted( - self, sample_providers, sample_address, sample_stamps, sample_token + self, sample_providers, sample_address, sample_stamps, sample_token, client ): bulk_payload = [] @@ -300,90 +558,3 @@ def test_get_address_from_did(self, sample_address): did = f"did:pkh:eip155:1:{sample_address}" address = get_address_from_did(did) assert address == sample_address - - -class TestComposeDBBulkStampUpdates: - base_url = "/ceramic-cache" - stamp_version = CeramicCache.StampType.V1 - - def test_success_bulk_update_compose_db_status( - self, - sample_providers, - sample_address, - sample_stamps, - sample_token, - ui_scorer, - ): - # Create stamps first - create_response = client.post( - f"{self.base_url}/stamps/bulk", - json.dumps( - [ - { - "provider": sample_providers[0], - "stamp": sample_stamps[0], - }, - { - "provider": sample_providers[1], - "stamp": sample_stamps[1], - }, - # This one will stay pending - { - "provider": sample_providers[2], - "stamp": sample_stamps[2], - }, - ] - ), - content_type="application/json", - **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, - ) - assert create_response.status_code == 201 - - ids = [stamp["id"] for stamp in create_response.json()["stamps"]] - - missing_id = 1234564564564564564 - - # Construct payload - bulk_payload = [ - { - "id": ids[0], - "compose_db_save_status": "saved", - "compose_db_stream_id": "stream-id-1", - }, - { - "id": ids[1], - "compose_db_save_status": "failed", - }, - # This is not valid, but should not interfere - { - "id": missing_id, - "compose_db_save_status": "saved", - }, - ] - - # Make the request - response = client.patch( - f"{self.base_url}/stamps/bulk/meta/compose-db", - json.dumps(bulk_payload), - content_type="application/json", - **{"HTTP_AUTHORIZATION": f"Bearer {sample_token}"}, - ) - - assert response.status_code == 200 - - data = response.json() - - assert len(data["updated"]) == 2 - - assert missing_id not in data["updated"] - assert ids[0] in data["updated"] - assert ids[1] in data["updated"] - - assert CeramicCache.objects.filter(compose_db_save_status="saved").count() == 1 - assert CeramicCache.objects.filter(compose_db_save_status="failed").count() == 1 - assert ( - CeramicCache.objects.filter(compose_db_save_status="pending").count() == 1 - ) - assert ( - CeramicCache.objects.filter(compose_db_stream_id="stream-id-1").count() == 1 - ) diff --git a/api/embed/api.py b/api/embed/api.py index 14573e6e1..3029bf909 100644 --- a/api/embed/api.py +++ b/api/embed/api.py @@ -11,6 +11,7 @@ GetStampsWithV2ScoreResponse, ) from ceramic_cache.api.v1 import handle_add_stamps_only +from ceramic_cache.models import CeramicCache from registry.api.schema import ( ErrorMessageResponse, ) @@ -61,10 +62,10 @@ class AddStampsPayload(Schema): def add_stamps( request, address: str, payload: AddStampsPayload ) -> GetStampsWithV2ScoreResponse: - return handle_add_stamps(address, payload) + return handle_embed_add_stamps(address, payload) -def handle_add_stamps( +def handle_embed_add_stamps( address: str, payload: AddStampsPayload ) -> GetStampsWithV2ScoreResponse: address_lower = address.lower() @@ -82,7 +83,7 @@ def handle_add_stamps( try: added_stamps = handle_add_stamps_only( - address, add_stamps_payload, payload.scorer_id + address, add_stamps_payload, CeramicCache.SourceApp.EMBED, payload.scorer_id ) user_account = Community.objects.get(id=payload.scorer_id).account score = async_to_sync(handle_scoring)(address, payload.scorer_id, user_account) diff --git a/api/embed/lambda_fn.py b/api/embed/lambda_fn.py index 98ee1c02d..02e4e8eb7 100644 --- a/api/embed/lambda_fn.py +++ b/api/embed/lambda_fn.py @@ -40,7 +40,7 @@ validate_address_and_convert_to_lowercase, ) -from .api import AccountAPIKeySchema, AddStampsPayload, handle_add_stamps +from .api import AccountAPIKeySchema, AddStampsPayload, handle_embed_add_stamps # pylint: enable=wrong-import-position @@ -143,7 +143,7 @@ def _handler_save_stamps(event, _context, body, _sensitive_date): _address = get_address(event["path"]) address_lower = validate_address_and_convert_to_lowercase(_address) - add_stamps_response = handle_add_stamps(address_lower, add_stamps_payload) + add_stamps_response = handle_embed_add_stamps(address_lower, add_stamps_payload) return { "statusCode": 200, diff --git a/api/embed/test/test_api_stamps.py b/api/embed/test/test_api_stamps.py index 018341f99..3837ca7ad 100644 --- a/api/embed/test/test_api_stamps.py +++ b/api/embed/test/test_api_stamps.py @@ -1,5 +1,6 @@ import json from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import cast from unittest.mock import patch @@ -7,10 +8,10 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager from django.test import Client, TestCase -from ninja_jwt.schema import RefreshToken from account.models import Account, AccountAPIKey, Community from ceramic_cache.models import CeramicCache +from registry.models import Score from registry.weight_models import WeightConfiguration, WeightConfigurationItem from scorer.settings.gitcoin_passport_weights import GITCOIN_PASSPORT_WEIGHTS from scorer_weighted.models import BinaryWeightedScorer, Scorer @@ -73,6 +74,7 @@ }, }, ] +mock_stamps = sorted(mock_stamps, key=lambda x: x["credentialSubject"]["provider"]) class StampsApiTestCase(TestCase): @@ -197,8 +199,6 @@ def test_submit_additional_valid_stamps(self, _test_submit_valid_stamps): self.assertEqual(stamps_response.status_code, 200) data = stamps_response.json() - cc = list(CeramicCache.objects.all()) - assert data["success"] == True last_score_timestamp = data["score"].pop("last_score_timestamp") @@ -237,7 +237,7 @@ def test_submit_additional_valid_stamps(self, _test_submit_valid_stamps): ) == sorted(mock_stamps, key=lambda x: x["credentialSubject"]["provider"]) @patch("registry.atasks.validate_credential", side_effect=[[], [], []]) - def test_submit_additional_valid_stamps(self, _test_submit_valid_stamps): + def test_storing_stamps_and_score(self, _test_submit_valid_stamps): """Existing stamps in the with the same providers are overriden and only counted once towards the score""" # Create the initial stamps in the DB @@ -258,7 +258,6 @@ def test_submit_additional_valid_stamps(self, _test_submit_valid_stamps): self.assertEqual(stamps_response.status_code, 200) data = stamps_response.json() - assert data["success"] == True last_score_timestamp = data["score"].pop("last_score_timestamp") @@ -295,3 +294,90 @@ def test_submit_additional_valid_stamps(self, _test_submit_valid_stamps): [d["stamp"] for d in data["stamps"]], key=lambda x: x["credentialSubject"]["provider"], ) == sorted(mock_stamps, key=lambda x: x["credentialSubject"]["provider"]) + + @patch("registry.atasks.validate_credential", side_effect=[[], [], []]) + def test_submitted_stamps_are_saved_properly(self, _test_submit_valid_stamps): + """Test that the newly submitted stamps are stored in the DB properly""" + + # Create the rest of the stamps via the POST request + stamps_response = self.client.post( + f"/embed/stamps/{mock_addresse}", + json.dumps({"scorer_id": self.community.id, "stamps": mock_stamps}), + content_type="application/json", + **{"HTTP_AUTHORIZATION": settings.CGRANTS_API_TOKEN}, + ) + self.assertEqual(stamps_response.status_code, 200) + + # Check the stamps stored in the DB + cc = list(CeramicCache.objects.all().values()) + assert len(cc) == len(mock_stamps) + for idx, c in enumerate(cc): + m = mock_stamps[idx] + assert ( + c + == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + # Here are the values we control + "address": mock_addresse, + "provider": m["credentialSubject"]["provider"], + "compose_db_save_status": "pending", # TODO: what state do we desire here ??? + "compose_db_stream_id": "", + "deleted_at": None, + "expiration_date": datetime.fromisoformat(m["expirationDate"]), + "issuance_date": datetime.fromisoformat(m["issuanceDate"]), + "proof_value": m["proof"]["proofValue"], + "source_app": CeramicCache.SourceApp.EMBED.value, + "source_scorer_id": self.community.id, + "stamp": m, + "type": 1, + } + ) + + # Check the score stored in the DB + scores = Score.objects.filter( + passport__address=mock_addresse, passport__community=self.community + ).values() + assert len(scores) == 1 + score = scores[0] + assert score == { + # Just copy the automatically generated values over + "id": score["id"], + "last_score_timestamp": score["last_score_timestamp"], + "passport_id": score["passport_id"], + # Here are the values we control + "error": None, + "evidence": { + "rawScore": "45", + "success": True, + "threshold": "20.00000", + "type": "ThresholdScoreCheck", + }, + "expiration_date": min(expiration_dates), + "score": Decimal("1.000000000"), + "stamp_scores": { + "Ens": 15.0, + "Gitcoin": 15.0, + "Google": 15.0, + }, + "stamps": { + "Ens": { + "dedup": False, + "expiration_date": expiration_dates[0].isoformat(), + "score": "15.00000", + }, + "Google": { + "dedup": False, + "expiration_date": expiration_dates[1].isoformat(), + "score": "15.00000", + }, + "Gitcoin": { + "dedup": False, + "expiration_date": expiration_dates[2].isoformat(), + "score": "15.00000", + }, + }, + "status": "DONE", + } diff --git a/api/embed/test/test_lambda_stamps.py b/api/embed/test/test_lambda_stamps.py index a07fcced4..35e7b504b 100644 --- a/api/embed/test/test_lambda_stamps.py +++ b/api/embed/test/test_lambda_stamps.py @@ -1,11 +1,12 @@ import json from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import cast from unittest.mock import patch from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager -from django.test import Client, TestCase +from django.test import TestCase from test_api_stamps import ( expiration_dates, mock_addresse, @@ -16,7 +17,9 @@ from account.models import Account, AccountAPIKey, Community from aws_lambdas.scorer_api_passport.tests.helpers import MockContext +from ceramic_cache.models import CeramicCache from embed.lambda_fn import lambda_handler_save_stamps +from registry.models import Score from registry.weight_models import WeightConfiguration, WeightConfigurationItem from scorer.settings.gitcoin_passport_weights import GITCOIN_PASSPORT_WEIGHTS from scorer_weighted.models import BinaryWeightedScorer, Scorer @@ -133,3 +136,105 @@ def test_rate_limit_success(self, _validate_credential, _close_old_connections): [d["stamp"] for d in data["stamps"]], key=lambda x: x["credentialSubject"]["provider"], ) == sorted(mock_stamps, key=lambda x: x["credentialSubject"]["provider"]) + + @patch("embed.lambda_fn.close_old_connections", side_effect=[None]) + @patch("registry.atasks.validate_credential", side_effect=[[], [], []]) + def test_storing_stamps_and_score( + self, _validate_credential, _close_old_connections + ): + """Test that the newly submitted stamps are stored in the DB properly""" + + (api_key_obj, api_key) = AccountAPIKey.objects.create_key( + account=self.account, + name="Token for user 1", + ) + + event = { + "headers": {"x-api-key": api_key}, + "path": f"/embed/stamps/{mock_addresse}", + "isBase64Encoded": False, + "body": json.dumps({"scorer_id": self.community.id, "stamps": mock_stamps}), + } + + result = lambda_handler_save_stamps(event, MockContext()) + + data = json.loads(result["body"]) + + assert result["headers"] == { + "Content-Type": "application/json", + } + assert result["statusCode"] == 200 + + # Check the stamps stored in the DB + cc = list(CeramicCache.objects.all().values()) + assert len(cc) == len(mock_stamps) + for idx, c in enumerate(cc): + m = mock_stamps[idx] + assert ( + c + == { + # Just copy the automatically generated values over + "id": c["id"], + "created_at": c["created_at"], + "updated_at": c["updated_at"], + # Here are the values we control + "address": mock_addresse, + "provider": m["credentialSubject"]["provider"], + "compose_db_save_status": "pending", # TODO: what state do we desire here ??? + "compose_db_stream_id": "", + "deleted_at": None, + "expiration_date": datetime.fromisoformat(m["expirationDate"]), + "issuance_date": datetime.fromisoformat(m["issuanceDate"]), + "proof_value": m["proof"]["proofValue"], + "source_app": CeramicCache.SourceApp.EMBED.value, + "source_scorer_id": self.community.id, + "stamp": m, + "type": 1, + } + ) + + # Check the score stored in the DB + scores = Score.objects.filter( + passport__address=mock_addresse, passport__community=self.community + ).values() + assert len(scores) == 1 + score = scores[0] + assert score == { + # Just copy the automatically generated values over + "id": score["id"], + "last_score_timestamp": score["last_score_timestamp"], + "passport_id": score["passport_id"], + # Here are the values we control + "error": None, + "evidence": { + "rawScore": "45", + "success": True, + "threshold": "20.00000", + "type": "ThresholdScoreCheck", + }, + "expiration_date": min(expiration_dates), + "score": Decimal("1.000000000"), + "stamp_scores": { + "Ens": 15.0, + "Gitcoin": 15.0, + "Google": 15.0, + }, + "stamps": { + "Ens": { + "dedup": False, + "expiration_date": expiration_dates[0].isoformat(), + "score": "15.00000", + }, + "Google": { + "dedup": False, + "expiration_date": expiration_dates[1].isoformat(), + "score": "15.00000", + }, + "Gitcoin": { + "dedup": False, + "expiration_date": expiration_dates[2].isoformat(), + "score": "15.00000", + }, + }, + "status": "DONE", + } diff --git a/api/poetry.lock b/api/poetry.lock index b0c695637..d2343a6be 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "abnf" @@ -2995,6 +2995,24 @@ pluggy = ">=1.5,<2" [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.25.2" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075"}, + {file = "pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f"}, +] + +[package.dependencies] +pytest = ">=8.2,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "pytest-bdd" version = "8.1.0" @@ -3963,4 +3981,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "23daf2757279acbff39b650319f4744a4d163be4fe2a7974e1cc2c12ad7c9608" +content-hash = "0d8e55d917fb7c0e4e031ec8f558e7e5f28b8ff625380821e2e6d0c4377835f8" diff --git a/api/pyproject.toml b/api/pyproject.toml index e22bccf6a..be5674bee 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -50,6 +50,7 @@ django-stubs = "*" pandas = "*" faker = "*" freezegun = "*" +pytest-asyncio = "^0.25.2" [tool.poetry.group.server.dependencies] diff --git a/api/registry/api/v1.py b/api/registry/api/v1.py index 154b43e36..a6199d882 100644 --- a/api/registry/api/v1.py +++ b/api/registry/api/v1.py @@ -1,9 +1,11 @@ -from typing import List, Optional +from typing import List from urllib.parse import urljoin import django_filters import requests +from asgiref.sync import async_to_sync from django.conf import settings +from django.contrib.auth import get_user_model from django.core.cache import cache from ninja import Router from ninja.pagination import paginate @@ -15,6 +17,7 @@ # --- Deduplication Modules from account.models import ( Account, + AccountAPIKey, Community, Nonce, Rules, @@ -59,7 +62,6 @@ ) from registry.filters import GTCStakeEventsFilter from registry.models import GTCStakeEvent, Passport, Score -from registry.tasks import score_passport_passport, score_registry_passport from registry.utils import ( decode_cursor, encode_cursor, @@ -68,6 +70,9 @@ reverse_lazy_with_query, ) +User = get_user_model() + + SCORE_TIMESTAMP_FIELD_DESCRIPTION = """ The optional `timestamp` query parameter can be used to retrieve the latest score for an address as of that timestamp. @@ -173,7 +178,9 @@ async def a_submit_passport( raise e except Exception as e: log.exception("Error submitting passport: %s", e) - raise InternalServerErrorException("Unexpected error while submitting passport") + raise InternalServerErrorException( + "Unexpected error while submitting passport" + ) from e async def ahandle_submit_passport( @@ -225,60 +232,9 @@ async def ahandle_submit_passport( def handle_submit_passport( - payload: SubmitPassportPayload, account: Account, use_passport_task: bool = False + payload: SubmitPassportPayload, account: Account ) -> DetailedScoreResponse: - address_lower = payload.address.lower() - - try: - scorer_id = get_scorer_id(payload) - except Exception as e: - raise e - - # Get community object - user_community = get_scorer_by_id(scorer_id, account) - - # Verify the signer - if payload.signature or community_requires_signature(user_community): - if get_signer(payload.nonce, payload.signature).lower() != address_lower: - raise InvalidSignerException() - - # Verify nonce - if not Nonce.use_nonce(payload.nonce): - log.error("Invalid nonce %s for address %s", payload.nonce, payload.address) - raise InvalidNonceException() - - # Create an empty passport instance, only needed to be able to create a pending Score - # The passport will be updated by the score_passport task - db_passport, _ = Passport.objects.update_or_create( - address=payload.address.lower(), - community=user_community, - defaults={ - "requires_calculation": True, - }, - ) - - # Create a score with status PROCESSING - score, _ = Score.objects.update_or_create( - passport_id=db_passport.pk, - defaults=dict(score=None, status=Score.Status.PROCESSING), - ) - - if use_passport_task: - score_passport_passport.delay(user_community.pk, payload.address) - else: - score_registry_passport.delay(user_community.pk, payload.address) - - return DetailedScoreResponse( - address=score.passport.address, - score=score.score, - status=score.status, - evidence=score.evidence, - last_score_timestamp=( - score.last_score_timestamp.isoformat() - if score.last_score_timestamp - else None - ), - ) + return async_to_sync(ahandle_submit_passport)(payload, account) def get_scorer_by_id(scorer_id: int | str, account: Account) -> Community: @@ -551,21 +507,31 @@ def get_passport_stamps( domain = request.build_absolute_uri("/")[:-1] next_url = ( - f"""{domain}{reverse_lazy_with_query( - "registry:get_passport_stamps", - args=[address], - query_kwargs={"token": encode_cursor(d="next", id=next_id), "limit": limit}, - )}""" + f"""{domain}{ + reverse_lazy_with_query( + "registry:get_passport_stamps", + args=[address], + query_kwargs={ + "token": encode_cursor(d="next", id=next_id), + "limit": limit, + }, + ) + }""" if has_more_stamps else None ) prev_url = ( - f"""{domain}{reverse_lazy_with_query( - "registry:get_passport_stamps", - args=[address], - query_kwargs={"token": encode_cursor(d="prev", id=prev_id), "limit": limit}, - )}""" + f"""{domain}{ + reverse_lazy_with_query( + "registry:get_passport_stamps", + args=[address], + query_kwargs={ + "token": encode_cursor(d="prev", id=prev_id), + "limit": limit, + }, + ) + }""" if has_prev_stamps else None ) diff --git a/api/registry/tasks.py b/api/registry/tasks.py index a8d250ad9..c5a7655fe 100644 --- a/api/registry/tasks.py +++ b/api/registry/tasks.py @@ -46,14 +46,6 @@ def save_api_key_analytics( log.error("Failed to save analytics. Error: '%s'", e, exc_info=True) -def score_passport_passport(community_id: int, address: str): - score_passport(community_id, address) - - -def score_registry_passport(community_id: int, address: str): - score_passport(community_id, address) - - def score_passport(community_id: int, address: str): passport = load_passport_record(community_id, address) diff --git a/api/registry/test/test_score_passport.py b/api/registry/test/test_score_passport.py index fbbfc0090..e43dee99a 100644 --- a/api/registry/test/test_score_passport.py +++ b/api/registry/test/test_score_passport.py @@ -5,15 +5,19 @@ from decimal import Decimal from unittest.mock import call, patch +import pytest +from asgiref.sync import async_to_sync from django.conf import settings from django.contrib.auth import get_user_model from django.test import Client, TransactionTestCase from web3 import Web3 from account.models import Account, AccountAPIKey, Community +from registry.api.schema import StatusEnum from registry.api.v1 import SubmitPassportPayload, a_submit_passport, get_score from registry.models import Event, HashScorerLink, Passport, Score, Stamp -from registry.tasks import score_passport_passport, score_registry_passport +from registry.tasks import score_passport +from scorer_weighted.models import Scorer, WeightedScorer User = get_user_model() my_mnemonic = settings.TEST_MNEMONIC @@ -27,6 +31,12 @@ now + timedelta(days=3), ] +mocked_weights = { + "Ens": 1.0, + "Google": 2.0, + "Gitcoin": 3.0, +} + mock_passport_data = { "stamps": [ { @@ -73,6 +83,9 @@ }, ] } +mock_min_expiration_date = min( + *[s["credential"]["expirationDate"] for s in mock_passport_data["stamps"]] +) def mock_validate(*args, **kwargs): @@ -103,21 +116,26 @@ def setUp(self): user=self.user, address=account.address ) - AccountAPIKey.objects.create_key( + (api_key_instance, _) = AccountAPIKey.objects.create_key( account=self.user_account, name="Token for user 1" ) + self.api_key = api_key_instance self.community = Community.objects.create( name="My Community", description="My Community description", account=self.user_account, + scorer=WeightedScorer.objects.create( + type=Scorer.Type.WEIGHTED, + weights=mocked_weights, + ), ) self.client = Client() def test_no_passport(self): with patch("registry.atasks.aget_passport", return_value=None): - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) passport = Passport.objects.get( address=self.account.address, community_id=self.community.pk @@ -144,16 +162,20 @@ def test_score_nonchecksummed_address(self): self._score_address(address) - async def _score_address(self, address): + def _score_address(self, address): class MockRequest: - def __init__(self, account): - self.auth = account - self.api_key = account.api_keys.all()[0] - - mock_request = MockRequest(self.user_account) - - with patch("registry.api.v1.score_passport_passport.delay", return_value=None): - await a_submit_passport( + def __init__(self): + pass + + mock_request = MockRequest() + mock_request.auth = self.user_account + mock_request.api_key = self.api_key + mock_request.path = "/passport/" + mock_request.GET = {} + mock_request.headers = {} + + with patch("registry.api.v1.ascore_passport", return_value=None): + async_to_sync(a_submit_passport)( mock_request, SubmitPassportPayload( address=address, @@ -165,13 +187,25 @@ def __init__(self, account): with patch( "registry.atasks.validate_credential", side_effect=mock_validate ): - score_passport_passport(self.community.pk, address) + score_passport(self.community.pk, address) Passport.objects.get(address=address, community_id=self.community.pk) score = get_score(mock_request, address, self.community.pk) - assert Decimal(score.score) == Decimal("3") - assert score.status == "DONE" + assert score.last_score_timestamp is not None + score_dict = score.model_dump() + assert score.model_dump() == { + # attributes set automatically + "last_score_timestamp": score_dict["last_score_timestamp"], + # attributes we expect to have a specific value + "address": self.account.address.lower(), + "evidence": None, + "error": None, + "score": Decimal("6.000000000"), + "status": StatusEnum.done, + "stamp_scores": mocked_weights, + "expiration_date": mock_min_expiration_date, + } def test_cleaning_stale_stamps(self): passport, _ = Passport.objects.update_or_create( @@ -194,7 +228,7 @@ def test_cleaning_stale_stamps(self): with patch( "registry.atasks.validate_credential", side_effect=mock_validate ): - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) my_stamps = Stamp.objects.filter(passport=passport) assert len(my_stamps) == 3 @@ -229,8 +263,8 @@ def test_deduplication_of_scoring_tasks(self): ): with patch("registry.tasks.log.info") as mock_log: # Call score_passport_passport twice, but only one of them should actually execute the scoring calculation - score_passport_passport(self.community.pk, self.account.address) - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) expected_call = call( "Passport no passport found for address='%s', community_id='%s' that has requires_calculation=True or None", @@ -256,121 +290,51 @@ def test_lifo_duplicate_stamp_scoring(self): requires_calculation=True, ) - passport_with_duplicates, _ = Passport.objects.update_or_create( - address=self.account_3.address, - community_id=self.community.pk, - requires_calculation=True, - ) + mocked_duplicate_stamps = {"stamps": mock_passport_data["stamps"][:1]} + mocked_non_duplicate_stamps = {"stamps": mock_passport_data["stamps"][1:]} - already_existing_stamp = { - "provider": "POAP", - "credential": { - "type": ["VerifiableCredential"], - "credentialSubject": { - "id": settings.TRUSTED_IAM_ISSUERS[0], - "hash": "0x1111", - "provider": "Gitcoin", + for stamp in mocked_duplicate_stamps["stamps"]: + Stamp.objects.update_or_create( + hash=stamp["credential"]["credentialSubject"]["hash"], + passport=passport_for_already_existing_stamp, + defaults={ + "provider": stamp["provider"], + "credential": json.dumps(stamp["credential"]), }, - "issuer": settings.TRUSTED_IAM_ISSUERS[0], - "issuanceDate": "2023-02-06T23:22:58.848Z", - "expirationDate": "2099-02-06T23:22:58.848Z", - }, - } - - Stamp.objects.update_or_create( - hash=already_existing_stamp["credential"]["credentialSubject"]["hash"], - passport=passport_for_already_existing_stamp, - defaults={ - "provider": already_existing_stamp["provider"], - "credential": json.dumps(already_existing_stamp["credential"]), - }, - ) - - HashScorerLink.objects.create( - hash=already_existing_stamp["credential"]["credentialSubject"]["hash"], - address=passport_for_already_existing_stamp.address, - community=passport_for_already_existing_stamp.community, - expires_at=already_existing_stamp["credential"]["expirationDate"], - ) + ) - mock_passport_data_with_duplicates = { - "stamps": [ - mock_passport_data["stamps"][0], - already_existing_stamp, - { - "provider": "Google", - "credential": { - "type": ["VerifiableCredential"], - "credentialSubject": { - "id": settings.TRUSTED_IAM_ISSUERS[0], - "hash": "0x12121", - "provider": "Google", - }, - "issuer": settings.TRUSTED_IAM_ISSUERS[0], - "issuanceDate": "2023-02-06T23:22:58.848Z", - "expirationDate": "2099-02-06T23:22:58.848Z", - }, - }, - ] - } + HashScorerLink.objects.create( + hash=stamp["credential"]["credentialSubject"]["hash"], + address=passport_for_already_existing_stamp.address, + community=passport_for_already_existing_stamp.community, + expires_at=stamp["credential"]["expirationDate"], + ) with patch("registry.atasks.validate_credential", side_effect=mock_validate): # Score original passport - with patch( - "registry.atasks.aget_passport", return_value=mock_passport_data - ): - score_registry_passport(self.community.pk, passport.address) - - assert ( - Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count() - == 0 - ) - - # Score passport with duplicates (one duplicate from original passport, - # one duplicate from already existing stamp) with patch( "registry.atasks.aget_passport", - return_value=mock_passport_data_with_duplicates, + return_value=mock_passport_data, ): - score_registry_passport( - self.community.pk, passport_with_duplicates.address - ) + score_passport(self.community.pk, passport.address) original_stamps = Stamp.objects.filter(passport=passport) - assert len(original_stamps) == 3 + assert len(original_stamps) == len(mocked_non_duplicate_stamps["stamps"]) - assert (Score.objects.get(passport=passport).score) == Decimal( - "0.933000000" - ) - - assert ( - Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count() - == 2 - ) - - deduplicated_stamps = Stamp.objects.filter( - passport=passport_with_duplicates - ) - assert len(deduplicated_stamps) == 1 - - assert ( - Score.objects.get(passport=passport_with_duplicates).score - ) == Decimal("0.525000000") - - passport.requires_calculation = True - passport.save() - # Re-score original passport, just to make sure it doesn't change - with patch( - "registry.atasks.aget_passport", return_value=mock_passport_data - ): - score_registry_passport(self.community.pk, passport.address) + assert Event.objects.filter( + action=Event.Action.LIFO_DEDUPLICATION + ).count() == len(mocked_duplicate_stamps["stamps"]) assert (Score.objects.get(passport=passport).score) == Decimal( - "0.933000000" + sum( + mocked_weights[s["credential"]["credentialSubject"]["provider"]] + for s in mocked_non_duplicate_stamps["stamps"] + ) ) - assert ( - Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count() - == 2 + + # All the stamps we worked with should be registered in HashScorerLink by now + assert HashScorerLink.objects.all().count() == len( + mock_passport_data["stamps"] ) def test_score_events(self): @@ -418,7 +382,7 @@ def test_score_expiration_time(self): with patch( "registry.atasks.validate_credential", side_effect=mock_validate ): - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) score = Score.objects.get(passport=passport) @@ -457,7 +421,7 @@ def test_score_expiration_time_when_all_stamps_expired(self): with patch( "registry.atasks.validate_credential", side_effect=mock_validate ): - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) score = Score.objects.get(passport=passport) @@ -481,7 +445,7 @@ def test_score_expiration_time_when_all_stamps_expired(self): with patch( "registry.atasks.validate_credential", side_effect=mock_validate ): - score_passport_passport(self.community.pk, self.account.address) + score_passport(self.community.pk, self.account.address) score = Score.objects.get(passport=passport) diff --git a/api/registry/utils.py b/api/registry/utils.py index cb1df9007..b58ab28a4 100644 --- a/api/registry/utils.py +++ b/api/registry/utils.py @@ -2,6 +2,7 @@ import json from datetime import datetime, timezone from functools import wraps +from typing import List from urllib.parse import urlencode import didkit @@ -27,7 +28,7 @@ def index(request): return render(request, "registry/index.html", context) -async def validate_credential(did, credential): +async def validate_credential(did, credential) -> List[str]: # pylint: disable=fixme stamp_return_errors = [] credential_subject = credential.get("credentialSubject") diff --git a/api/scorer/test/test_choose_binary_scorer.py b/api/scorer/test/test_choose_binary_scorer.py index 4ad179e52..0f6081102 100644 --- a/api/scorer/test/test_choose_binary_scorer.py +++ b/api/scorer/test/test_choose_binary_scorer.py @@ -10,7 +10,7 @@ from pytest_bdd import given, scenario, then, when from account.models import Community -from registry.tasks import score_passport_passport +from registry.tasks import score_passport from registry.test.test_passport_submission import mock_passport, mock_utc_timestamp from registry.weight_models import WeightConfiguration from scorer_weighted.models import BinaryWeightedScorer @@ -169,7 +169,7 @@ def _(scorer_community_with_binary_scorer, scorer_api_key): ) # execute the task - score_passport_passport( + score_passport( scorer_community_with_binary_scorer.id, "0x71ad3e3057ca74967239c66ca6d3a9c2a43a58fc", ) diff --git a/api/tox.ini b/api/tox.ini index 51f2d680f..2b974b614 100644 --- a/api/tox.ini +++ b/api/tox.ini @@ -3,5 +3,8 @@ DJANGO_SETTINGS_MODULE = scorer.settings # -- recommended but optional: python_files = tests.py test_*.py *_tests.py +asyncio_mode=auto +asyncio_default_fixture_loop_scope="module" + [flake8] max-line-length = 160 diff --git a/api/v2/test/test_api_dedup.py b/api/v2/test/test_api_dedup.py index 194d211ac..38c9db027 100644 --- a/api/v2/test/test_api_dedup.py +++ b/api/v2/test/test_api_dedup.py @@ -1,12 +1,10 @@ import copy from datetime import datetime, timedelta, timezone from decimal import Decimal -from re import M from unittest.mock import patch import pytest from django.conf import settings -from django.contrib.auth.models import User from django.test import Client from web3 import Web3