Fix fixture tests (probably? Needs new fixtures on udata-fixtures)

udata/commands/fixtures.py

@@ -39,7 +39,7 @@
 DISCUSSION_URL = "/api/1/discussions"
 
 
-DEFAULT_FIXTURE_FILE_TAG: str = "v4.0.0"
+DEFAULT_FIXTURE_FILE_TAG: str = "v5.0.0"
 DEFAULT_FIXTURE_FILE: str = f"https://raw.githubusercontent.com/opendatateam/udata-fixtures/{DEFAULT_FIXTURE_FILE_TAG}/results.json"  # noqa
 
 DEFAULT_FIXTURES_RESULTS_FILENAME: str = "results.json"
@@ -113,11 +113,17 @@ def generate_fixtures_file(data_source: str, results_filename: str) -> None:
         for slug in bar:
             json_fixture = {}
 
-            json_dataset = requests.get(f"{data_source}{DATASET_URL}/{slug}/").json()
+            url = f"{data_source}{DATASET_URL}/{slug}/"
+            response = requests.get(url)
+            if not response.ok:
+                print(f"Got a status code {response.status_code} while getting {url}, skipping")
+                continue
+            json_dataset = response.json()
             json_resources = json_dataset.pop("resources")
             if json_dataset["organization"] is None:
                 json_owner = json_dataset.pop("owner")
-                json_dataset["owner"] = json_owner["id"]
+                if json_owner:
+                    json_dataset["owner"] = json_owner["id"]
             else:
                 json_org = json_dataset.pop("organization")
                 json_org = requests.get(f"{data_source}{ORG_URL}/{json_org['id']}/").json()
@@ -138,6 +144,7 @@ def generate_fixtures_file(data_source: str, results_filename: str) -> None:
             json_discussion = requests.get(
                 f"{data_source}{DISCUSSION_URL}/?for={json_dataset['id']}"
             ).json()["data"]
+
             json_fixture["discussions"] = json_discussion
 
             json_dataservices = requests.get(
@@ -152,8 +159,16 @@ def generate_fixtures_file(data_source: str, results_filename: str) -> None:
         print(f"Fixtures saved to file {results_filename}")
 
 
-def get_or_create(data, key, model, factory):
+def get_or_create_obj(data, model, factory):
     """Try getting the object. If it doesn't exist yet, create it with the provided factory."""
+    obj = model.objects(id=data["id"]).first()
+    if not obj:
+        obj = factory(**data)
+    return obj
+
+
+def get_or_create(data, key, model, factory):
+    """Try getting the object from data[key]. If it doesn't exist yet, create it with the provided factory."""
     if key not in data or data[key] is None:
         return
     data[key] = remove_unwanted_keys(data[key], key)
@@ -192,6 +207,12 @@ def import_fixtures(source):
             user = UserFactory()
             dataset = fixture["dataset"]
             dataset = remove_unwanted_keys(dataset, "dataset")
+            contact_points = []
+            for contact_point in dataset.get("contact_points"):
+                contact_points.append(
+                    get_or_create_obj(contact_point, ContactPoint, ContactPointFactory)
+                )
+            dataset["contact_points"] = contact_points
             if fixture["organization"]:
                 organization = fixture["organization"]
                 organization["members"] = [
@@ -229,9 +250,11 @@ def import_fixtures(source):
                 DiscussionFactory(**discussion, subject=dataset)
             for dataservice in fixture["dataservices"]:
                 dataservice = remove_unwanted_keys(dataservice, "dataservice")
-                # TODO: update fixtures
-                dataservice["contact_points"] = [
-                    get_or_create(dataservice, "contact_points", ContactPoint, ContactPointFactory)
-                ]
+                contact_points = []
+                for contact_point in dataservice.get("contact_points"):
+                    contact_points.append(
+                        get_or_create_obj(contact_point, ContactPoint, ContactPointFactory)
+                    )
+                dataservice["contact_points"] = contact_points
                 dataservice["organization"] = get_or_create_organization(dataservice)
                 DataserviceFactory(**dataservice, datasets=[dataset])

udata/commands/tests/test_fixtures.py

@@ -6,6 +6,7 @@
 
 import udata.commands.fixtures
 from udata import models
+from udata.core.contact_point.factories import ContactPointFactory
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataset.factories import (
     CommunityResourceFactory,
@@ -31,9 +32,12 @@ def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
         org = OrganizationFactory(
             members=[Member(user=user, role="editor"), Member(user=admin, role="admin")]
         )
+        contact_point = ContactPointFactory(role="contact")
         # Set the same slug we're 'exporting' from the FIXTURE_DATASET_SLUG config, see the
         # @pytest.mark.options above.
-        dataset = DatasetFactory(slug="some-test-dataset-slug", organization=org)
+        dataset = DatasetFactory(
+            slug="some-test-dataset-slug", organization=org, contact_points=[contact_point]
+        )
         res = ResourceFactory()
         dataset.add_resource(res)
         ReuseFactory(datasets=[dataset], owner=user)
@@ -48,12 +52,13 @@ def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
             ],
             closed_by=admin,
         )
-        DataserviceFactory(datasets=[dataset], organization=org)
+        DataserviceFactory(datasets=[dataset], organization=org, contact_points=[contact_point])
 
         with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
             # Get the fixtures from the local instance.
             monkeypatch.setattr(requests, "get", lambda url: api.get(url))
             monkeypatch.setattr(Response, "json", Response.get_json)
+            Response.ok = True
             result = cli("generate-fixtures-file", "", fixtures_fd.name)
             fixtures_fd.flush()
             assert "Fixtures saved to file " in result.output
@@ -65,13 +70,15 @@ def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
             models.CommunityResource.drop_collection()
             models.User.drop_collection()
             models.Dataservice.drop_collection()
+            models.ContactPoint.drop_collection()
 
             assert models.Organization.objects(slug=org.slug).count() == 0
             assert models.Dataset.objects.count() == 0
             assert models.Discussion.objects.count() == 0
             assert models.CommunityResource.objects.count() == 0
             assert models.User.objects.count() == 0
             assert models.Dataservice.objects.count() == 0
+            assert models.ContactPoint.objects.count() == 0
 
             # Then load them in the database to make sure they're correct.
             result = cli("import-fixtures", fixtures_fd.name)
@@ -82,6 +89,8 @@ def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
         assert result_org.members[1].user.id == admin.id
         assert result_org.members[1].role == "admin"
         assert models.Dataset.objects.count() > 0
+        result_dataset = models.Dataset.objects.first()
+        assert result_dataset.contact_points == [contact_point]
         assert models.Discussion.objects.count() > 0
         result_discussion = models.Discussion.objects.first()
         assert result_discussion.user.id == user.id
@@ -95,6 +104,7 @@ def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
         # Make sure we also import the dataservice organization
         result_dataservice = models.Dataservice.objects.first()
         assert result_dataservice.organization == org
+        assert result_dataservice.contact_points == [contact_point]
 
     def test_import_fixtures_from_default_file(self, cli):
         """Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""