Merge pull request #32 from archesproject/jtw/formatting-ci

Add formatting step to CI

.github/workflows/main.yml

@@ -61,6 +61,11 @@ jobs:
             echo "package.json not found, skipping npm commands."
           fi
 
+      - name: Check formatting
+        run: |
+          npm run prettier:check
+          black . --check --exclude=node_modules
+
       - name: Run frontend tests
         run: |
           npm run vitest

.prettierrc

@@ -1,3 +1,4 @@
 {
-    "singleAttributePerLine": true
+    "singleAttributePerLine": true,
+    "tabWidth": 4
 }

arches_lingo/celery.py

@@ -7,7 +7,7 @@
 if platform.system().lower() == "windows":
     os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1")
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'arches_lingo.settings')
-app = Celery('arches_lingo')
-app.config_from_object('django.conf:settings', namespace='CELERY')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "arches_lingo.settings")
+app = Celery("arches_lingo")
+app.config_from_object("django.conf:settings", namespace="CELERY")
 app.autodiscover_tasks()

arches_lingo/etl_modules/migrate_to_lingo.py

@@ -7,7 +7,10 @@
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.etl_modules.save import save_to_tiles
 from arches.app.etl_modules.decorators import load_data_async
-from arches.app.etl_modules.base_import_module import BaseImportModule, FileValidationError
+from arches.app.etl_modules.base_import_module import (
+    BaseImportModule,
+    FileValidationError,
+)
 from arches.app.models import models
 from arches.app.models.concept import Concept
 from arches.app.models.models import LoadStaging, NodeGroup, LoadEvent
@@ -47,17 +50,24 @@ def __init__(self, request=None, loadid=None):
         self.moduleid = request.POST.get("module") if request else None
         self.loadid = request.POST.get("loadid") if request else loadid
         self.datatype_factory = DataTypeFactory()
-    
+
     def etl_schemes(self, cursor, nodegroup_lookup, node_lookup):
         schemes = []
-        for concept in models.Concept.objects.filter(nodetype="ConceptScheme").prefetch_related("value_set"):
+        for concept in models.Concept.objects.filter(
+            nodetype="ConceptScheme"
+        ).prefetch_related("value_set"):
             scheme_to_load = {"type": "Scheme", "tile_data": []}
             for value in concept.value_set.all():
-                scheme_to_load["resourceinstanceid"] = concept.pk # use old conceptid as new resourceinstanceid
+                scheme_to_load["resourceinstanceid"] = (
+                    concept.pk
+                )  # use old conceptid as new resourceinstanceid
 
                 name = {}
                 identifier = {}
-                if value.valuetype_id == "prefLabel" or value.valuetype_id == "altLabel":
+                if (
+                    value.valuetype_id == "prefLabel"
+                    or value.valuetype_id == "altLabel"
+                ):
                     name["name_content"] = value.value
                     name["name_language"] = value.language_id
                     name["name_type"] = value.valuetype_id
@@ -67,18 +77,25 @@ def etl_schemes(self, cursor, nodegroup_lookup, node_lookup):
                     identifier["identifier_type"] = value.valuetype_id
                     scheme_to_load["tile_data"].append({"identifier": identifier})
             schemes.append(scheme_to_load)
-        self.populate_staging_table(cursor, schemes, nodegroup_lookup, node_lookup)       
+        self.populate_staging_table(cursor, schemes, nodegroup_lookup, node_lookup)
 
     def etl_concepts(self, cursor, nodegroup_lookup, node_lookup):
         concepts = []
-        for concept in models.Concept.objects.filter(nodetype="Concept").prefetch_related("value_set"):
+        for concept in models.Concept.objects.filter(
+            nodetype="Concept"
+        ).prefetch_related("value_set"):
             concept_to_load = {"type": "Concept", "tile_data": []}
             for value in concept.value_set.all():
-                concept_to_load["resourceinstanceid"] = concept.pk # use old conceptid as new resourceinstanceid
+                concept_to_load["resourceinstanceid"] = (
+                    concept.pk
+                )  # use old conceptid as new resourceinstanceid
 
                 name = {}
                 identifier = {}
-                if value.valuetype_id == "prefLabel" or value.valuetype_id == "altLabel":
+                if (
+                    value.valuetype_id == "prefLabel"
+                    or value.valuetype_id == "altLabel"
+                ):
                     name["name_content"] = value.value
                     name["name_language"] = value.language_id
                     name["name_type"] = value.valuetype_id
@@ -90,8 +107,9 @@ def etl_concepts(self, cursor, nodegroup_lookup, node_lookup):
             concepts.append(concept_to_load)
         self.populate_staging_table(cursor, concepts, nodegroup_lookup, node_lookup)
 
-
-    def populate_staging_table(self, cursor, concepts_to_load, nodegroup_lookup, node_lookup):
+    def populate_staging_table(
+        self, cursor, concepts_to_load, nodegroup_lookup, node_lookup
+    ):
         tiles_to_load = []
         for concept_to_load in concepts_to_load:
             for mock_tile in concept_to_load["tile_data"]:
@@ -100,23 +118,32 @@ def populate_staging_table(self, cursor, concepts_to_load, nodegroup_lookup, nod
                 nodegroup_depth = nodegroup_lookup[nodegroup_id]["depth"]
                 tile_id = uuid.uuid4()
                 parent_tile_id = None
-                tile_value_json, passes_validation = self.create_tile_value(cursor, mock_tile, nodegroup_alias, nodegroup_lookup, node_lookup)
+                tile_value_json, passes_validation = self.create_tile_value(
+                    cursor, mock_tile, nodegroup_alias, nodegroup_lookup, node_lookup
+                )
                 operation = "insert"
-                tiles_to_load.append(LoadStaging(
-                    load_event=LoadEvent(self.loadid),
-                    nodegroup=NodeGroup(nodegroup_id),
-                    resourceid=concept_to_load["resourceinstanceid"],
-                    tileid=tile_id,
-                    parenttileid=parent_tile_id,
-                    value=tile_value_json,
-                    nodegroup_depth=nodegroup_depth,
-                    source_description="{0}: {1}".format(concept_to_load["type"], nodegroup_alias),  # source_description
-                    passes_validation=passes_validation,
-                    operation=operation,
-                ))
+                tiles_to_load.append(
+                    LoadStaging(
+                        load_event=LoadEvent(self.loadid),
+                        nodegroup=NodeGroup(nodegroup_id),
+                        resourceid=concept_to_load["resourceinstanceid"],
+                        tileid=tile_id,
+                        parenttileid=parent_tile_id,
+                        value=tile_value_json,
+                        nodegroup_depth=nodegroup_depth,
+                        source_description="{0}: {1}".format(
+                            concept_to_load["type"], nodegroup_alias
+                        ),  # source_description
+                        passes_validation=passes_validation,
+                        operation=operation,
+                    )
+                )
         staged_tiles = LoadStaging.objects.bulk_create(tiles_to_load)
-
-        cursor.execute("""CALL __arches_check_tile_cardinality_violation_for_load(%s)""", [self.loadid])
+
+        cursor.execute(
+            """CALL __arches_check_tile_cardinality_violation_for_load(%s)""",
+            [self.loadid],
+        )
         cursor.execute(
             """
                 INSERT INTO load_errors (type, source, error, loadid, nodegroupid)
@@ -127,7 +154,9 @@ def populate_staging_table(self, cursor, concepts_to_load, nodegroup_lookup, nod
             [self.loadid],
         )
 
-    def create_tile_value(self, cursor, mock_tile, nodegroup_alias, nodegroup_lookup, node_lookup):
+    def create_tile_value(
+        self, cursor, mock_tile, nodegroup_alias, nodegroup_lookup, node_lookup
+    ):
         tile_value = {}
         tile_valid = True
         for node_alias in mock_tile[nodegroup_alias].keys():
@@ -140,8 +169,10 @@ def create_tile_value(self, cursor, mock_tile, nodegroup_alias, nodegroup_lookup
                 config = node_details["config"]
                 config["loadid"] = self.loadid
                 config["nodeid"] = nodeid
-
-                value, validation_errors = self.prepare_data_for_loading(datatype_instance, source_value, config)
+
+                value, validation_errors = self.prepare_data_for_loading(
+                    datatype_instance, source_value, config
+                )
                 valid = True if len(validation_errors) == 0 else False
                 if not valid:
                     tile_valid = False
@@ -150,18 +181,34 @@ def create_tile_value(self, cursor, mock_tile, nodegroup_alias, nodegroup_lookup
                     error_message = error["message"]
                     cursor.execute(
                         """INSERT INTO load_errors (type, value, source, error, message, datatype, loadid, nodeid) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
-                        ("node", source_value, "", error["title"], error["message"], datatype, self.loadid, nodeid),
+                        (
+                            "node",
+                            source_value,
+                            "",
+                            error["title"],
+                            error["message"],
+                            datatype,
+                            self.loadid,
+                            nodeid,
+                        ),
                     )
-
-                tile_value[nodeid] = {"value": value, "valid": valid, "source": source_value, "notes": error_message, "datatype": datatype}
+
+                tile_value[nodeid] = {
+                    "value": value,
+                    "valid": valid,
+                    "source": source_value,
+                    "notes": error_message,
+                    "datatype": datatype,
+                }
             except KeyError:
-                pass 
+                pass
 
         return tile_value, tile_valid
-    
+
     def init_relationships(self, cursor, loadid):
         # Create top concept of scheme relationships (derived from relations with 'hasTopConcept' relationtype)
-        cursor.execute("""
+        cursor.execute(
+            """
            insert into load_staging(
                 value,
                 resourceid,
@@ -193,10 +240,17 @@ def init_relationships(self, cursor, loadid):
                 'insert' as operation
             from relations
             where relationtype = 'hasTopConcept';
-        """, (CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID, loadid, CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID))
+        """,
+            (
+                CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID,
+                loadid,
+                CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID,
+            ),
+        )
 
         # Create broader relationships (derived from relations with 'narrower' relationtype)
-        cursor.execute("""
+        cursor.execute(
+            """
            insert into load_staging(
                 value,
                 resourceid,
@@ -228,11 +282,14 @@ def init_relationships(self, cursor, loadid):
                 'insert' as operation
             from relations
             where relationtype = 'narrower';
-        """, (CONCEPTS_BROADER_NODEGROUP_ID, loadid, CONCEPTS_BROADER_NODEGROUP_ID))
+        """,
+            (CONCEPTS_BROADER_NODEGROUP_ID, loadid, CONCEPTS_BROADER_NODEGROUP_ID),
+        )
 
         # Create Part of Scheme relationships - derived by recursively generating concept hierarchy & associating
         # concepts with their schemes
-        cursor.execute("""
+        cursor.execute(
+            """
            insert into load_staging(
                 value,
                 resourceid,
@@ -274,14 +331,28 @@ def init_relationships(self, cursor, loadid):
                 %s::uuid as nodegroupid,
                 'insert' as operation
             FROM concept_hierarchy;
-        """, (CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID, loadid, CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID))
+        """,
+            (
+                CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID,
+                loadid,
+                CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID,
+            ),
+        )
 
     def start(self, request):
         load_details = {"operation": "RDM to Lingo Migration"}
         cursor = connection.cursor()
         cursor.execute(
             """INSERT INTO load_event (loadid, complete, status, etl_module_id, load_details, load_start_time, user_id) VALUES (%s, %s, %s, %s, %s, %s, %s)""",
-            (self.loadid, False, "running", self.moduleid, json.dumps(load_details), datetime.now(), self.userid),
+            (
+                self.loadid,
+                False,
+                "running",
+                self.moduleid,
+                json.dumps(load_details),
+                datetime.now(),
+                self.userid,
+            ),
         )
         message = "load event created"
         return {"success": True, "data": message}
@@ -290,22 +361,26 @@ def write(self, request):
         self.loadid = request.POST.get("loadid")
         if models.Concept.objects.count() < 500:
             response = self.run_load_task(self.userid, self.loadid)
-        else: 
+        else:
             response = self.run_load_task_async(request, self.loadid)
         message = "Schemes and Concept Migration to Lingo Models Complete"
         return {"success": True, "data": message}
 
     def run_load_task(self, userid, loadid):
         self.loadid = loadid  # currently redundant, but be certain
-        
+
         with connection.cursor() as cursor:
 
             # Gather and load schemes and concepts
-            schemes_nodegroup_lookup, schemes_nodes = self.get_graph_tree(SCHEMES_GRAPH_ID)
+            schemes_nodegroup_lookup, schemes_nodes = self.get_graph_tree(
+                SCHEMES_GRAPH_ID
+            )
             schemes_node_lookup = self.get_node_lookup(schemes_nodes)
             self.etl_schemes(cursor, schemes_nodegroup_lookup, schemes_node_lookup)
 
-            concepts_nodegroup_lookup, concepts_nodes = self.get_graph_tree(CONCEPTS_GRAPH_ID)
+            concepts_nodegroup_lookup, concepts_nodes = self.get_graph_tree(
+                CONCEPTS_GRAPH_ID
+            )
             concepts_node_lookup = self.get_node_lookup(concepts_nodes)
             self.etl_concepts(cursor, concepts_nodegroup_lookup, concepts_node_lookup)
 
@@ -320,11 +395,13 @@ def run_load_task(self, userid, loadid):
                     ("validated", loadid),
                 )
                 response = save_to_tiles(userid, loadid)
-                cursor.execute("""CALL __arches_update_resource_x_resource_with_graphids();""")
+                cursor.execute(
+                    """CALL __arches_update_resource_x_resource_with_graphids();"""
+                )
                 cursor.execute("""SELECT __arches_refresh_spatial_views();""")
                 refresh_successful = cursor.fetchone()[0]
                 if not refresh_successful:
-                    raise Exception('Unable to refresh spatial views')
+                    raise Exception("Unable to refresh spatial views")
                 return response
             else:
                 cursor.execute(

arches_lingo/install/requirements_dev.txt

@@ -1,5 +1,6 @@
 livereload
 sst
 coverage
-sauceclient
-django-silk==5.1.0
+django-silk==5.1.0
+pre-commit
+black==24.4.2

arches_lingo/migrations/0001_initial.py

@@ -23,15 +23,17 @@ def add_plugins(apps, schema_editor):
             componentname="reference-data-manager",
             config={},
             slug="reference-data-manager",
-            sortorder=0
+            sortorder=0,
         )
 
     def remove_plugin(apps, schema_editor):
         Plugin = apps.get_model("models", "Plugin")
 
-        for plugin in Plugin.objects.filter(pluginid__in=["29321ce0-bd95-4357-a2a5-822e9cb06f70"]):
+        for plugin in Plugin.objects.filter(
+            pluginid__in=["29321ce0-bd95-4357-a2a5-822e9cb06f70"]
+        ):
             plugin.delete()
 
     operations = [
         migrations.RunPython(add_plugins, remove_plugin),
-    ]
+    ]

arches_lingo/search_indexes/sample_index.py

@@ -3,8 +3,18 @@
 
 class SampleIndex(BaseIndex):
     def prepare_index(self):
-        self.index_metadata = {"mappings": {"properties": {"tile_count": {"type": "keyword"}, "graph_id": {"type": "keyword"}}}}
+        self.index_metadata = {
+            "mappings": {
+                "properties": {
+                    "tile_count": {"type": "keyword"},
+                    "graph_id": {"type": "keyword"},
+                }
+            }
+        }
         super(SampleIndex, self).prepare_index()
 
     def get_documents_to_index(self, resourceinstance, tiles):
-        return ({"tile_count": len(tiles), "graph_id": resourceinstance.graph_id}, str(resourceinstance.resourceinstanceid))
+        return (
+            {"tile_count": len(tiles), "graph_id": resourceinstance.graph_id},
+            str(resourceinstance.resourceinstanceid),
+        )

arches_lingo/src/App.vue

@@ -4,4 +4,4 @@ import Foo from "@/components/FooComponent.vue";
 
 <template>
     <Foo />
-</template>
+</template>