Opitimalization: tables relationship (join) (#334)

This commit add some speed up with modifying "lazy" parameter in db.relationship. Mostly is added JOIN type. This reduce a quantity of database requests from framework resulting more user gui fluently work and reducing waiting times on some actions.

    added lazy="joined" and lazy="selectin" (Attribute, AttributeEnum, NewsItem, NewsItemData, AttributeGroupItem, ReportItemAttribute, User)
    replaced some manual code filtration with JOIN relationship
    speed up attributes creation on report (when adding new)
    .gitgnore: add .editorconfig file

.gitignore

@@ -48,6 +48,7 @@ local/
 *.njsproj.user
 *.sln
 *.sw?
+.editorconfig
 
 # ignore custom templates
 src/presenters/templates/custom/*

src/core/model/attribute.py

@@ -7,7 +7,8 @@
 import os
 from xml.etree.ElementTree import iterparse
 from marshmallow import fields, post_load
-from sqlalchemy import orm, func, or_
+from sqlalchemy import orm, func, or_, and_
+from sqlalchemy.orm import backref
 
 from managers import log_manager
 from managers.db_manager import db
@@ -48,7 +49,7 @@ class AttributeEnum(db.Model):
     imported = db.Column(db.Boolean, default=False)
 
     attribute_id = db.Column(db.Integer, db.ForeignKey("attribute.id"))
-    attribute = db.relationship("Attribute")
+    attribute = db.relationship("Attribute", back_populates="attribute_enums")
 
     def __init__(self, id, index, value, description):
         """Initialize the attribute enum.
@@ -306,7 +307,6 @@ class Attribute(db.Model):
         find_by_type: Finds an attribute by type.
         get: Retrieves attributes based on search criteria.
         get_all_json: Retrieves all attributes in JSON format.
-        get_enums: Retrieves attribute enums for a given attribute.
         create_attribute: Creates a new attribute.
         add_attribute: Adds a new attribute.
         update: Updates an attribute.
@@ -325,6 +325,15 @@ class Attribute(db.Model):
     validator = db.Column(db.Enum(AttributeValidator))
     validator_parameter = db.Column(db.String())
 
+    attribute_enums = db.relationship("AttributeEnum",
+        primaryjoin=and_(
+            id == AttributeEnum.attribute_id,
+            or_(
+                type == AttributeType.RADIO, type == AttributeType.ENUM
+            )
+        ),
+        back_populates="attribute", lazy="subquery")
+
     def __init__(self, id, name, description, type, default_value, validator, validator_parameter, attribute_enums):
         """Initialize an Attribute object.
 
@@ -448,21 +457,6 @@ def get_all_json(cls, search):
         attribute_schema = AttributePresentationSchema(many=True)
         return {"total_count": total_count, "items": attribute_schema.dump(attributes)}
 
-    @classmethod
-    def get_enums(cls, attribute):
-        """Retrieve attribute enums for a given attribute.
-
-        Args:
-            attribute (Attribute): The attribute object.
-
-        Returns:
-            list: A list of attribute enums.
-        """
-        if attribute.type == AttributeType.RADIO or attribute.type == AttributeType.ENUM:
-            return AttributeEnum.get_all_for_attribute(attribute.id)
-        else:
-            return []
-
     @classmethod
     def create_attribute(cls, attribute):
         """Create a new attribute.

src/core/model/news_item.py

@@ -45,7 +45,7 @@ class NewsItemData(db.Model):
     published = db.Column(db.String())
     updated = db.Column(db.DateTime, default=datetime.now())
 
-    attributes = db.relationship('NewsItemAttribute', secondary="news_item_data_news_item_attribute")
+    attributes = db.relationship('NewsItemAttribute', secondary="news_item_data_news_item_attribute", lazy='selectin')
 
     osint_source_id = db.Column(db.String, db.ForeignKey('osint_source.id'), nullable=True)
     osint_source = db.relationship('OSINTSource')
@@ -172,7 +172,7 @@ class NewsItem(db.Model):
     relevance = db.Column(db.Integer, default=0)
 
     news_item_data_id = db.Column(db.String, db.ForeignKey('news_item_data.id'))
-    news_item_data = db.relationship('NewsItemData')
+    news_item_data = db.relationship('NewsItemData', lazy='selectin')
 
     news_item_aggregate_id = db.Column(db.Integer, db.ForeignKey('news_item_aggregate.id'))
 
@@ -398,9 +398,9 @@ class NewsItemAggregate(db.Model):
 
     osint_source_group_id = db.Column(db.String, db.ForeignKey('osint_source_group.id'))
 
-    news_items = db.relationship("NewsItem")
+    news_items = db.relationship("NewsItem", lazy='joined')
 
-    news_item_attributes = db.relationship("NewsItemAttribute", secondary='news_item_aggregate_news_item_attribute')
+    news_item_attributes = db.relationship("NewsItemAttribute", secondary='news_item_aggregate_news_item_attribute', lazy='selectin')
 
     @classmethod
     def find(cls, news_item_aggregate_id):

src/core/model/report_item.py

@@ -115,11 +115,11 @@ class ReportItemAttribute(db.Model):
     current = db.Column(db.Boolean, default=True)
 
     attribute_group_item_id = db.Column(db.Integer, db.ForeignKey("attribute_group_item.id"))
-    attribute_group_item = db.relationship("AttributeGroupItem", viewonly=True)
+    attribute_group_item = db.relationship("AttributeGroupItem", viewonly=True, lazy="joined", order_by=AttributeGroupItem.index)
     attribute_group_item_title = db.Column(db.String)
 
     report_item_id = db.Column(db.Integer, db.ForeignKey("report_item.id"), nullable=True)
-    report_item = db.relationship("ReportItem")
+    report_item = db.relationship("ReportItem", back_populates="attributes")
 
     user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=True)
     user = db.relationship("User")
@@ -261,7 +261,7 @@ class ReportItem(db.Model):
         secondaryjoin=ReportItemRemoteReportItem.remote_report_item_id == id,
     )
 
-    attributes = db.relationship("ReportItemAttribute", back_populates="report_item", cascade="all, delete-orphan")
+    attributes = db.relationship("ReportItemAttribute", back_populates="report_item", cascade="all, delete-orphan", lazy="joined")
 
     report_item_cpes = db.relationship("ReportItemCpe", cascade="all, delete-orphan")
 
@@ -697,7 +697,7 @@ def update_report_item(cls, id, data, user):
 
                 if "attribute_id" in data:
                     for attribute in report_item.attributes:
-                        # Compare attribute IDs
+                        # convert ID to string, we compare types: int & int or int & str
                         if str(attribute.id) == str(data["attribute_id"]):
                             if attribute.value != data["attribute_value"]:
                                 modified = True
@@ -737,7 +737,7 @@ def update_report_item(cls, id, data, user):
                 if "attribute_id" in data:
                     attribute_to_delete = None
                     for attribute in report_item.attributes:
-                        # sometime we compare: int & int or int & str
+                        # convert ID to string, we compare types: int & int or int & str
                         if str(attribute.id) == str(data["attribute_id"]):
                             attribute_to_delete = attribute
                             break
@@ -945,8 +945,8 @@ def update_cpes(self):
         self.report_item_cpes = []
         if self.completed is True:
             for attribute in self.attributes:
-                attribute_group = AttributeGroupItem.find(attribute.attribute_group_item_id)
-                if attribute_group.attribute.type == AttributeType.CPE:
+                item = AttributeGroupItem.find(attribute.attribute_group_item_id)
+                if item.attribute.type == AttributeType.CPE:
                     self.report_item_cpes.append(ReportItemCpe(attribute.value))
 
 

src/core/model/report_item_type.py

@@ -28,10 +28,10 @@ class AttributeGroupItem(db.Model):
     max_occurrence = db.Column(db.Integer)
 
     attribute_group_id = db.Column(db.Integer, db.ForeignKey('attribute_group.id'))
-    attribute_group = db.relationship("AttributeGroup", viewonly=True)
+    attribute_group = db.relationship("AttributeGroup", back_populates="attribute_group_items", viewonly=True, lazy="joined")
 
     attribute_id = db.Column(db.Integer, db.ForeignKey('attribute.id'))
-    attribute = db.relationship("Attribute")
+    attribute = db.relationship("Attribute", lazy="joined")
 
     def __init__(self, id, title, description, index, min_occurrence, max_occurrence, attribute_id):
         if id is not None and id != -1:
@@ -50,10 +50,6 @@ def __init__(self, id, title, description, index, min_occurrence, max_occurrence
     def find(cls, id):
         return cls.query.get(id)
 
-    @staticmethod
-    def sort(attribute_group_item):
-        return attribute_group_item.index
-
 
 class NewAttributeGroupSchema(AttributeGroupBaseSchema):
     attribute_group_items = fields.Nested('NewAttributeGroupItemSchema', many=True)
@@ -76,7 +72,7 @@ class AttributeGroup(db.Model):
     report_item_type = db.relationship("ReportItemType")
 
     attribute_group_items = db.relationship('AttributeGroupItem', back_populates="attribute_group",
-                                            cascade="all, delete-orphan")
+                                            cascade="all, delete-orphan", lazy="joined", order_by=AttributeGroupItem.index)
 
     def __init__(self, id, title, description, section, section_title, index, attribute_group_items):
         if id is not None and id != -1:
@@ -91,14 +87,6 @@ def __init__(self, id, title, description, section, section_title, index, attrib
         self.index = index
         self.attribute_group_items = attribute_group_items
 
-    @orm.reconstructor
-    def reconstruct(self):
-        self.attribute_group_items.sort(key=AttributeGroupItem.sort)
-
-    @staticmethod
-    def sort(attribute_group):
-        return attribute_group.index
-
     def update(self, updated_attribute_group):
         self.title = updated_attribute_group.title
         self.description = updated_attribute_group.description
@@ -148,7 +136,7 @@ class ReportItemType(db.Model):
     description = db.Column(db.String())
 
     attribute_groups = db.relationship('AttributeGroup', back_populates="report_item_type",
-                                       cascade="all, delete-orphan")
+                                       cascade="all, delete-orphan", lazy="joined", order_by=AttributeGroup.index)
 
     def __init__(self, id, title, description, attribute_groups):
         self.id = None
@@ -162,7 +150,6 @@ def __init__(self, id, title, description, attribute_groups):
     def reconstruct(self):
         self.subtitle = self.description
         self.tag = "mdi-file-table-outline"
-        self.attribute_groups.sort(key=AttributeGroup.sort)
 
     @classmethod
     def find(cls, id):
@@ -200,16 +187,11 @@ def get(cls, search, user, acl_check):
                 func.lower(ReportItemType.title).like(search_string),
                 func.lower(ReportItemType.description).like(search_string)))
 
-        return query.order_by(db.asc(ReportItemType.title)).all(), query.count()
+        return query.order_by(ReportItemType.title).all(), query.count()
 
     @classmethod
     def get_all_json(cls, search, user, acl_check):
         report_item_types, count = cls.get(search, user, acl_check)
-        for report_item_type in report_item_types:
-            for attribute_group in report_item_type.attribute_groups:
-                for attribute_group_item in attribute_group.attribute_group_items:
-                    attribute_group_item.attribute.attribute_enums = Attribute.get_enums(
-                        attribute_group_item.attribute)
 
         report_item_type_schema = ReportItemTypePresentationSchema(many=True)
         return {'total_count': count, 'items': report_item_type_schema.dump(report_item_types)}

src/core/model/user.py

@@ -38,10 +38,10 @@ class User(db.Model):
 
     organizations = db.relationship("Organization", secondary="user_organization")
     roles = db.relationship(Role, secondary='user_role')
-    permissions = db.relationship(Permission, secondary='user_permission')
+    permissions = db.relationship(Permission, secondary='user_permission', lazy='joined')
 
     profile_id = db.Column(db.Integer, db.ForeignKey('user_profile.id'))
-    profile = db.relationship("UserProfile", cascade="all")
+    profile = db.relationship("UserProfile", cascade="all", lazy='joined')
 
     def __init__(self, id, username, name, password, organizations, roles, permissions):
         self.id = None
@@ -273,8 +273,8 @@ class UserProfile(db.Model):
     spellcheck = db.Column(db.Boolean, default=True)
     dark_theme = db.Column(db.Boolean, default=False)
     language = db.Column(db.String(2))
-    hotkeys = db.relationship("Hotkey", cascade="all, delete-orphan")
-    word_lists = db.relationship('WordList', secondary='user_profile_word_list')
+    hotkeys = db.relationship("Hotkey", cascade="all, delete-orphan", lazy='joined')
+    word_lists = db.relationship('WordList', secondary='user_profile_word_list', lazy='joined')
 
     def __init__(self, spellcheck, dark_theme, language, hotkeys, word_lists):
         self.id = None

src/core/model/word_list.py

@@ -38,7 +38,7 @@ class WordList(db.Model):
     description = db.Column(db.String(), nullable=False)
     use_for_stop_words = db.Column(db.Boolean, default=False)
 
-    categories = db.relationship("WordListCategory", cascade="all, delete-orphan")
+    categories = db.relationship("WordListCategory", cascade="all, delete-orphan", lazy='joined')
 
     def __init__(self, id, name, description, categories, use_for_stop_words):
         self.id = None
@@ -148,7 +148,7 @@ class WordListCategory(db.Model):
 
     word_list_id = db.Column(db.Integer, db.ForeignKey('word_list.id'))
 
-    entries = db.relationship("WordListEntry", cascade="all, delete-orphan")
+    entries = db.relationship("WordListEntry", cascade="all, delete-orphan", lazy='joined')
 
     def __init__(self, name = None, description = None, link = None, entries = None):
         self.id = None

src/presenters/api/presenters.py

@@ -13,6 +13,7 @@ def get(self):
 
     @api_key_required
     def post(self):
+        # print("=== GENERATE FROM THE FOLLOWING JSON ===", request.json, flush=True)
         return presenters_manager.generate(request.json)
 
 

src/shared/shared/schema/attribute.py

@@ -69,7 +69,7 @@ def make_attribute_enum(self, data, **kwargs):
 class AttributeEnum:
     """Class representing an attribute enum."""
 
-    def __init__(self, index, value, description):
+    def __init__(self, id, index, value, description):
         """
         Initialize an Attribute object.
 
@@ -78,6 +78,7 @@ def __init__(self, index, value, description):
             value (str): The value of the attribute.
             description (str): The description of the attribute.
         """
+        self.id = id
         self.index = index
         self.value = value
         self.description = description
@@ -120,7 +121,7 @@ class AttributePresentationSchema(AttributeSchema, PresentationSchema):
 class Attribute:
     """Class representing an attribute."""
 
-    def __init__(self, id, name, description, type, default_value, validator, validator_parameter):
+    def __init__(self, id, name, description, type, default_value, validator, validator_parameter, attribute_enums):
         """
         Initialize an Attribute object.
 
@@ -132,6 +133,7 @@ def __init__(self, id, name, description, type, default_value, validator, valida
             default_value: The default value of the attribute.
             validator: The validator function for the attribute.
             validator_parameter: The parameter for the validator function.
+            attribute_enums: Attribute enum values.
 
         Returns:
             None
@@ -143,3 +145,4 @@ def __init__(self, id, name, description, type, default_value, validator, valida
         self.default_value = default_value
         self.validator = validator
         self.validator_parameter = validator_parameter
+        self.attribute_enums = attribute_enums