insights: add a+ insights logic and display on project detail, also a…

…dd project result page

changelog/8875.md

@@ -0,0 +1,3 @@
+### Added
+- logic & model for insights from adhocracy+ used
+- project result page 

meinberlin/apps/projects/insights.py

@@ -0,0 +1,111 @@
+from typing import Iterable
+from typing import List
+
+from django.core.exceptions import FieldDoesNotExist
+from django.db.models import Q
+
+from adhocracy4.comments.models import Comment
+from adhocracy4.polls.models import Answer
+from adhocracy4.polls.models import Poll
+from adhocracy4.polls.models import Vote
+from adhocracy4.projects.models import Project
+from adhocracy4.ratings.models import Rating
+from meinberlin.apps.budgeting.models import Proposal
+from meinberlin.apps.ideas.models import Idea
+from meinberlin.apps.likes.models import Like
+from meinberlin.apps.livequestions.models import LiveQuestion
+from meinberlin.apps.mapideas.models import MapIdea
+from meinberlin.apps.projects.models import ProjectInsight
+from meinberlin.apps.topicprio.models import Topic
+
+
+def create_insights(projects: Iterable[Project]) -> List[ProjectInsight]:
+    return [create_insight(project=project) for project in projects]
+
+
+def create_insight(project: Project) -> ProjectInsight:
+    modules = project.modules.all()
+
+    ideas = Idea.objects.filter(module__in=modules)
+    map_ideas = MapIdea.objects.filter(module__in=modules)
+    comments = Comment.objects.filter(
+        Q(project=project) | Q(parent_comment__project=project)
+    )
+    proposals = Proposal.objects.filter(module__in=modules)
+    polls = Poll.objects.filter(module__in=modules)
+    votes = Vote.objects.filter(choice__question__poll__in=polls)
+    answers = Answer.objects.filter(question__poll__in=polls)
+    live_questions = LiveQuestion.objects.filter(module__in=modules)
+    likes = Like.objects.filter(question__in=live_questions)
+    topics = Topic.objects.filter(module__in=modules)
+
+    values = [Rating.POSITIVE, Rating.NEGATIVE]
+    ratings_ideas = Rating.objects.filter(idea__in=ideas, value__in=values)
+    ratings_map_ideas = Rating.objects.filter(mapidea__in=map_ideas, value__in=values)
+    ratings_comments = Rating.objects.filter(comment__in=comments, value__in=values)
+    ratings_topics = Rating.objects.filter(topic__in=topics, value__in=values)
+
+    creator_objects = [
+        comments,
+        ratings_comments,
+        ratings_ideas,
+        ratings_map_ideas,
+        ratings_topics,
+        ideas,
+        map_ideas,
+        votes,
+        answers,
+        proposals,
+    ]
+
+    rating_objects = [
+        ratings_comments,
+        ratings_map_ideas,
+        ratings_topics,
+        ratings_ideas,
+        likes,
+    ]
+
+    idea_objects = [ideas, map_ideas, proposals, topics]
+
+    insight, _ = ProjectInsight.objects.get_or_create(project=project)
+
+    insight.comments = comments.count()
+    insight.ratings = sum(x.count() for x in rating_objects)
+    insight.written_ideas = sum(x.count() for x in idea_objects)
+    insight.poll_answers = votes.count() + answers.count()
+    insight.live_questions = live_questions.count()
+    insight.save()
+
+    insight.active_participants.clear()
+    unregistered_participants = set()
+
+    for obj in creator_objects:
+        # ignore objects which don't have a creator, they are counted in the next step.
+        ids = list(
+            obj.filter(creator__isnull=False)
+            .values_list("creator", flat=True)
+            .distinct()
+            .order_by()
+        )
+        insight.active_participants.add(*ids)
+        # content from unregistered users doesn't have a creator but a content_id
+        if model_field_exists(obj.model, "content_id"):
+            content_ids = set(
+                obj.filter(content_id__isnull=False)
+                .values_list("content_id", flat=True)
+                .distinct()
+                .order_by()
+            )
+            unregistered_participants = unregistered_participants.union(content_ids)
+
+    insight.unregistered_participants = len(unregistered_participants)
+    return insight
+
+
+def model_field_exists(cls, field):
+    try:
+        cls._meta.get_field(field)
+        return True
+    except FieldDoesNotExist:
+        return False

meinberlin/apps/projects/management/commands/reset_insights_table.py

@@ -0,0 +1,39 @@
+from argparse import ArgumentParser
+
+from django.core.management.base import BaseCommand
+
+from meinberlin.apps import logger
+from meinberlin.apps.projects.insights import create_insights
+from meinberlin.apps.projects.models import Project
+
+
+class Command(BaseCommand):
+    help = "Resets the insights and participation tables."
+
+    def add_arguments(self, parser: ArgumentParser):
+        parser.add_argument(
+            "--project",
+            help="project slug, resets data for this project only",
+        )
+
+    def handle(self, *args, **options):
+        slug = options["project"]
+
+        if slug:
+            project = Project.objects.filter(slug=slug).first()
+            if not project:
+                known = Project.objects.order_by("slug").values_list("slug", flat=True)
+                logger.warning(f"unknown project slug: {slug=}, {list(known)=}")
+                return
+
+            projects = [project]
+        else:
+            projects = Project.objects.all()
+
+        if not projects:
+            logger.info("no projects found")
+            return
+
+        insights = create_insights(projects=projects)
+
+        logger.info(f"created insights: {len(projects)=}, {len(insights)=}")

meinberlin/apps/projects/migrations/0006_projectinsight.py

@@ -0,0 +1,67 @@
+# Generated by Django 4.2.16 on 2025-02-20 14:21
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("a4projects", "0047_alter_project_image_alter_project_tile_image"),
+        ("meinberlin_projects", "0005_topics"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ProjectInsight",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    models.DateTimeField(
+                        default=django.utils.timezone.now,
+                        editable=False,
+                        verbose_name="Created",
+                    ),
+                ),
+                (
+                    "modified",
+                    models.DateTimeField(
+                        blank=True, editable=False, null=True, verbose_name="Modified"
+                    ),
+                ),
+                ("unregistered_participants", models.PositiveIntegerField(default=0)),
+                ("comments", models.PositiveIntegerField(default=0)),
+                ("ratings", models.PositiveIntegerField(default=0)),
+                ("written_ideas", models.PositiveIntegerField(default=0)),
+                ("poll_answers", models.PositiveIntegerField(default=0)),
+                ("live_questions", models.PositiveIntegerField(default=0)),
+                (
+                    "active_participants",
+                    models.ManyToManyField(to=settings.AUTH_USER_MODEL),
+                ),
+                (
+                    "project",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="insight",
+                        to="a4projects.project",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

meinberlin/apps/projects/migrations/0007_create_insights.py

@@ -0,0 +1,111 @@
+import logging
+from collections import defaultdict
+from django.db import migrations
+
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_insights(apps, schema_editor):
+    ProjectInsight = apps.get_model("meinberlin_projects", "ProjectInsight")
+    Comment = apps.get_model("a4comments", "Comment")
+    Answer = apps.get_model("a4polls", "Answer")
+    Vote = apps.get_model("a4polls", "Vote")
+    Rating = apps.get_model("a4ratings", "Rating")
+    Proposal = apps.get_model("meinberlin_budgeting", "Proposal")
+    Idea = apps.get_model("meinberlin_ideas", "Idea")
+    Like = apps.get_model("meinberlin_likes", "Like")
+    LiveQuestion = apps.get_model("meinberlin_livequestions", "LiveQuestion")
+    MapIdea = apps.get_model("meinberlin_mapideas", "MapIdea")
+
+    insights = defaultdict(ProjectInsight)
+    user_ids = defaultdict(set)
+
+    for idea_model in [Idea, MapIdea, Proposal]:
+        for idea_object in idea_model.objects.all().select_related(
+            "creator", "module__project"
+        ):
+            project = idea_object.module.project
+            insights[project].written_ideas += 1
+            user_ids[project].add(idea_object.creator.id)
+
+    for live_question in LiveQuestion.objects.all().select_related("module__project"):
+        insights[live_question.module.project].live_questions += 1
+
+    for answer in Answer.objects.all().select_related(
+        "creator",
+        "question__poll__module__project",
+    ):
+        project = answer.question.poll.module.project
+        insights[project].poll_answers += 1
+        user_ids[project].add(answer.creator.id)
+
+    for vote in Vote.objects.all().select_related(
+        "creator",
+        "choice__question__poll__module__project",
+    ):
+        project = vote.choice.question.poll.module.project
+        insights[project].poll_answers += 1
+        user_ids[project].add(vote.creator.id)
+
+    for comment in Comment.objects.exclude(project=None).select_related(
+        "project", "creator"
+    ):
+        project = comment.project
+        insights[project].comments += 1
+        user_ids[project].add(comment.creator.id)
+
+    for like in Like.objects.all().select_related("question__module__project"):
+        project = like.question.module.project
+        insights[project].ratings += 1
+
+    for rating in Rating.objects.all().select_related("creator", "content_type"):
+        model_name = rating.content_type.model
+        content_model = apps.get_model(
+            app_label=rating.content_type.app_label,
+            model_name=model_name,
+        )
+        content_object = content_model.objects.get(id=rating.object_pk)
+
+        if model_name == "comment":
+            project = content_object.project
+        elif model_name in ["idea", "mapidea", "topic", "proposal"]:
+            project = content_object.module.project
+        else:
+            logger.warning(f"could not identify project: {rating=}")
+            continue
+
+        insights[project].ratings += 1
+        user_ids[project].add(rating.creator.id)
+
+    for project, insight in insights.items():
+        insight.project = project
+        insight.save()
+
+        if user_ids[project]:
+            insight.active_participants.add(*user_ids[project])
+
+
+def delete_insights(apps, schema_editor):
+    ProjectInsight = apps.get_model("meinberlin_projects", "ProjectInsight")
+    ProjectInsight.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("meinberlin_projects", "0006_projectinsight"),
+        ("a4polls", "0006_alter_answer_unique_together_answer_content_id_and_more"),
+        (
+            "a4ratings",
+            "0005_rename_rating_content_type_object_pk_a4ratings_r_content_8fb00e_idx",
+        ),
+        ("meinberlin_topicprio", "0014_alter_topic_description"),
+        (
+            "a4comments",
+            "0014_rename_comment_content_type_object_pk_a4comments__content_ff606b_idx",
+        ),
+    ]
+
+    operations = [
+        migrations.RunPython(code=initialize_insights, reverse_code=delete_insights),
+    ]