Merge branch 'main' into Improve-NestBot-/users-command

backend/.env.example

@@ -1,5 +1,6 @@
 DJANGO_ALGOLIA_APPLICATION_ID=None
 DJANGO_ALGOLIA_APPLICATION_REGION=None
+DJANGO_ALGOLIA_EXCLUDED_LOCAL_INDEX_NAMES=None
 DJANGO_ALGOLIA_WRITE_API_KEY=None
 DJANGO_ALLOWED_HOSTS=*
 DJANGO_AWS_ACCESS_KEY_ID=None
@@ -11,9 +12,9 @@ DJANGO_DB_PASSWORD=None
 DJANGO_DB_PORT=None
 DJANGO_DB_USER=None
 DJANGO_OPEN_AI_SECRET_KEY=None
+DJANGO_RELEASE_VERSION=None
 DJANGO_SECRET_KEY=None
 DJANGO_SENTRY_DSN=None
-DJANGO_RELEASE_VERSION=None
 DJANGO_SLACK_BOT_TOKEN=None
 DJANGO_SLACK_SIGNING_SECRET=None
 GITHUB_TOKEN=None

backend/apps/common/index.py

@@ -1,32 +1,137 @@
-"""Algolia index synonyms support and record count."""
+"""Algolia index common classes and helpers."""
 
 import logging
 from functools import lru_cache
 from pathlib import Path
 
 from algoliasearch.http.exceptions import AlgoliaException
+from algoliasearch.query_suggestions.client import QuerySuggestionsClientSync
 from algoliasearch.search.client import SearchClientSync
+from algoliasearch_django import AlgoliaIndex
+from algoliasearch_django.decorators import register as algolia_register
 from django.conf import settings
 
 from apps.common.constants import NL
 
 logger = logging.getLogger(__name__)
 
+EXCLUDED_LOCAL_INDEX_NAMES = (
+    "chapters_suggestions",
+    "committees_suggestions",
+    "issues_suggestions",
+    "projects_contributors_count_asc",
+    "projects_contributors_count_desc",
+    "projects_forks_count_asc",
+    "projects_forks_count_desc",
+    "projects_name_asc",
+    "projects_name_desc",
+    "projects_query_suggestions",
+    "projects_stars_count_asc",
+    "projects_stars_count_desc",
+    "projects_suggestions",
+    "users_suggestions",
+)
 IS_LOCAL_BUILD = settings.ENVIRONMENT == "Local"
 LOCAL_INDEX_LIMIT = 1000
 
 
-class IndexBase:
-    """Nest index synonyms mixin and record count."""
+class IndexRegistry:
+    """Registry to track and manage Algolia indices."""
+
+    _instance = None
+
+    def __init__(self):
+        """Initialize index registry."""
+        self.excluded_local_index_names = set()
+        self.load_excluded_local_index_names()
+
+    @classmethod
+    def get_instance(cls):
+        """Get or create a singleton instance of IndexRegistry."""
+        if cls._instance is None:
+            cls._instance = IndexRegistry()
+        return cls._instance
+
+    def is_indexable(self, name: str):
+        """Check if index is on."""
+        return name.lower() not in self.excluded_local_index_names if IS_LOCAL_BUILD else True
+
+    def load_excluded_local_index_names(self):
+        """Load excluded local index names."""
+        excluded_names = settings.ALGOLIA_EXCLUDED_LOCAL_INDEX_NAMES
+        self.excluded_local_index_names = set(
+            (
+                excluded_name.strip().lower()
+                for excluded_name in excluded_names.strip().split(",")
+                if excluded_name.strip()
+            )
+            if excluded_names and excluded_names != "None"
+            else EXCLUDED_LOCAL_INDEX_NAMES
+        )
+
+        return self
+
+
+def is_indexable(index_name: str):
+    """Determine if an index should be created based on configuration."""
+    return IndexRegistry.get_instance().is_indexable(index_name)
+
+
+def register(model, **kwargs):
+    """Register index if configuration allows."""
+
+    def wrapper(index_cls):
+        return (
+            algolia_register(model, **kwargs)(index_cls)
+            if is_indexable(f"{index_cls.index_name}")
+            else index_cls
+        )
+
+    return wrapper
+
+
+class IndexBase(AlgoliaIndex):
+    """Base index class."""
 
     @staticmethod
     def get_client():
-        """Get the Algolia client."""
+        """Return an instance of search client."""
         return SearchClientSync(
             settings.ALGOLIA_APPLICATION_ID,
             settings.ALGOLIA_WRITE_API_KEY,
         )
 
+    @staticmethod
+    def get_suggestions_client():
+        """Get suggestions client."""
+        return QuerySuggestionsClientSync(
+            settings.ALGOLIA_APPLICATION_ID,
+            settings.ALGOLIA_WRITE_API_KEY,
+            getattr(settings, "ALGOLIA_APPLICATION_REGION", None),
+        )
+
+    @staticmethod
+    def configure_replicas(index_name: str, replicas: dict):
+        """Configure replicas."""
+        if not is_indexable(index_name):
+            return  # Skip replicas configuration if base index is off.
+
+        env = settings.ENVIRONMENT.lower()
+
+        if indexable_replicas := {
+            f"{env}_{index_name}_{replica_name}": replica_ranking
+            for replica_name, replica_ranking in replicas.items()
+            if is_indexable(f"{index_name}_{replica_name}")
+        }:
+            client = IndexBase.get_client()
+            client.set_settings(
+                f"{env}_{index_name}",
+                {"replicas": sorted(indexable_replicas.keys())},
+            )
+
+            for replica_name, replica_ranking in indexable_replicas.items():
+                client.set_settings(replica_name, {"ranking": replica_ranking})
+
     @staticmethod
     def _parse_synonyms_file(file_path):
         """Parse synonyms file."""
@@ -94,9 +199,9 @@ def get_total_count(index_name, search_filters=None):
         client = IndexBase.get_client()
         try:
             search_params = {
-                "query": "",
-                "hitsPerPage": 0,
                 "analytics": False,
+                "hitsPerPage": 0,
+                "query": "",
             }
             if search_filters:
                 search_params["filters"] = search_filters
@@ -107,3 +212,9 @@ def get_total_count(index_name, search_filters=None):
         except AlgoliaException:
             logger.exception("Error retrieving index count for '%s'", index_name)
             return 0
+
+    def get_queryset(self):
+        """Get queryset."""
+        qs = self.get_entities()
+
+        return qs[:LOCAL_INDEX_LIMIT] if IS_LOCAL_BUILD else qs

backend/apps/common/management/commands/algolia_update_suggestions.py

@@ -1,22 +1,16 @@
 """A command to update OWASP Nest suggestions index."""
 
-from algoliasearch.query_suggestions.client import QuerySuggestionsClientSync
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
+from apps.common.index import IndexBase, is_indexable
+
 
 class Command(BaseCommand):
     help = "Create query suggestions for Algolia indices"
 
     def handle(self, *args, **kwargs):
-        if settings.ENVIRONMENT == "Local":
-            return
-
-        client = QuerySuggestionsClientSync(
-            settings.ALGOLIA_APPLICATION_ID,
-            settings.ALGOLIA_WRITE_API_KEY,
-            settings.ALGOLIA_APPLICATION_REGION,
-        )
+        client = IndexBase.get_suggestions_client()
 
         entity_configs = {
             "chapters": {
@@ -90,20 +84,21 @@ def handle(self, *args, **kwargs):
         }
 
         print("\nThe following query suggestion index were updated:")
+        environment = settings.ENVIRONMENT.lower()
         for entity, suggestion_settings in entity_configs.items():
-            source_index_name = f"{settings.ENVIRONMENT.lower()}_{entity}"
-            suggestions_index_name = f"{settings.ENVIRONMENT.lower()}_{entity}_suggestions"
+            if not is_indexable(entity) or not is_indexable(f"{entity}_suggestions"):
+                continue  # Skip if the index name is excluded.
 
             configuration = {
                 "sourceIndices": [
                     {
-                        "indexName": source_index_name,
+                        "indexName": f"{environment}_{entity}",
                         **suggestion_settings,
                     }
                 ]
             }
             client.update_config(
-                index_name=suggestions_index_name,
+                index_name=f"{environment}_{entity}_suggestions",
                 configuration=configuration,
             )
             print(f"{7*' '} * {entity.capitalize()}")

backend/apps/github/admin.py

@@ -1,4 +1,4 @@
-"""Repository app admin."""
+"""GitHub app admin."""
 
 from django.contrib import admin
 from django.utils.safestring import mark_safe

backend/apps/github/index/issue.py

@@ -1,14 +1,11 @@
 """GitHub issue index."""
 
-from algoliasearch_django import AlgoliaIndex
-from algoliasearch_django.decorators import register
-
-from apps.common.index import IS_LOCAL_BUILD, LOCAL_INDEX_LIMIT, IndexBase
+from apps.common.index import IndexBase, register
 from apps.github.models.issue import Issue
 
 
 @register(Issue)
-class IssueIndex(AlgoliaIndex, IndexBase):
+class IssueIndex(IndexBase):
     """Issue index."""
 
     index_name = "issues"
@@ -79,19 +76,17 @@ class IssueIndex(AlgoliaIndex, IndexBase):
 
     should_index = "is_indexable"
 
-    def get_queryset(self):
-        """Get queryset."""
-        qs = Issue.open_issues.assignable.select_related(
+    @staticmethod
+    def update_synonyms():
+        """Update synonyms."""
+        return IndexBase.reindex_synonyms("github", "issues")
+
+    def get_entities(self):
+        """Get entities for indexing."""
+        return Issue.open_issues.assignable.select_related(
             "repository",
         ).prefetch_related(
             "assignees",
             "labels",
             "repository__project_set",
         )
-
-        return qs[:LOCAL_INDEX_LIMIT] if IS_LOCAL_BUILD else qs
-
-    @staticmethod
-    def update_synonyms():
-        """Update synonyms."""
-        return IssueIndex.reindex_synonyms("github", "issue")

backend/apps/github/index/release.py

@@ -1,14 +1,11 @@
 """GitHub release Algolia index configuration."""
 
-from algoliasearch_django import AlgoliaIndex
-from algoliasearch_django.decorators import register
-
-from apps.common.index import IS_LOCAL_BUILD, LOCAL_INDEX_LIMIT, IndexBase
+from apps.common.index import IndexBase, register
 from apps.github.models.release import Release
 
 
 @register(Release)
-class ReleaseIndex(AlgoliaIndex, IndexBase):
+class ReleaseIndex(IndexBase):
     """Release index."""
 
     index_name = "releases"
@@ -53,15 +50,14 @@ class ReleaseIndex(AlgoliaIndex, IndexBase):
 
     should_index = "is_indexable"
 
-    def get_queryset(self):
-        """Get queryset for indexing."""
-        qs = Release.objects.filter(
-            is_draft=False,
-            published_at__isnull=False,
-        )
-        return qs[:LOCAL_INDEX_LIMIT] if IS_LOCAL_BUILD else qs
-
     @staticmethod
     def update_synonyms():
         """Update synonyms."""
         ReleaseIndex.reindex_synonyms("github", "releases")
+
+    def get_entities(self):
+        """Get entities for indexing."""
+        return Release.objects.filter(
+            is_draft=False,
+            published_at__isnull=False,
+        )

backend/apps/github/index/repository.py

@@ -1,14 +1,11 @@
 """GitHub repository Algolia index configuration."""
 
-from algoliasearch_django import AlgoliaIndex
-from algoliasearch_django.decorators import register
-
-from apps.common.index import IS_LOCAL_BUILD, LOCAL_INDEX_LIMIT, IndexBase
+from apps.common.index import IndexBase, register
 from apps.github.models.repository import Repository
 
 
 @register(Repository)
-class RepositoryIndex(AlgoliaIndex, IndexBase):
+class RepositoryIndex(IndexBase):
     """Repository index."""
 
     index_name = "repositories"
@@ -59,14 +56,15 @@ class RepositoryIndex(AlgoliaIndex, IndexBase):
 
     should_index = "is_indexable"
 
-    def get_queryset(self):
-        """Get queryset for indexing."""
-        qs = Repository.objects.filter(is_template=False).prefetch_related(
-            "repositorycontributor_set"
-        )
-        return qs[:LOCAL_INDEX_LIMIT] if IS_LOCAL_BUILD else qs
-
     @staticmethod
     def update_synonyms():
         """Update synonyms."""
         RepositoryIndex.reindex_synonyms("github", "repositories")
+
+    def get_entities(self):
+        """Get entities for indexing."""
+        return Repository.objects.filter(
+            is_template=False,
+        ).prefetch_related(
+            "repositorycontributor_set",
+        )