Merge branch 'main' into court-parent-documentation

.github/workflows/tests.yml

@@ -49,23 +49,6 @@ jobs:
       matrix:
         tag_flags: ["--exclude-tag selenium", "--tag selenium"]
     steps:
-      - name: Check out solr
-        uses: actions/checkout@v4
-        with:
-          repository: freelawproject/courtlistener-solr-server
-          ref: main
-          path: courtlistener-solr-server
-      - name: Set up solr permissions
-        run: |
-          cd courtlistener-solr-server
-          sudo chown -R :1024 data
-          sudo chown -R :1024 solr
-          sudo find data -type d -exec chmod g+s {} \;
-          sudo find solr -type d -exec chmod g+s {} \;
-          sudo find data -type d -exec chmod 775 {} \;
-          sudo find solr -type d -exec chmod 775 {} \;
-          sudo find data -type f -exec chmod 664 {} \;
-          sudo find solr -type f -exec chmod 664 {} \;
       - name: Check out CourtListener
         uses: actions/checkout@v4
         with:

cl/alerts/tests/tests.py

@@ -2392,7 +2392,7 @@ def test_es_alert_update_and_delete(self, mock_abort_audio):
             user=self.user_profile.user,
             rate=Alert.REAL_TIME,
             name="Test Alert OA",
-            query="type=oa&docket_number=19-1010",
+            query="type=oa&docket_number=19-1010&order_by=score desc",
             alert_type=SEARCH_TYPES.ORAL_ARGUMENT,
         )
 
@@ -2402,6 +2402,7 @@ def test_es_alert_update_and_delete(self, mock_abort_audio):
         response_str = str(doc.to_dict())
         self.assertIn("'query': '19-1010'", response_str)
         self.assertIn("'rate': 'rt'", response_str)
+        self.assertNotIn("function_score", response_str)
 
         # Update Alert
         search_alert_1.query = "type=oa&docket_number=19-1020"

cl/alerts/tests/tests_recap_alerts.py

@@ -2221,13 +2221,19 @@ def test_index_and_delete_recap_alerts_from_percolator(
             user=self.user_profile.user,
             rate=Alert.WEEKLY,
             name="Test Alert Docket Only",
-            query='q="401 Civil"&type=r',
+            query='q="401 Civil"&type=r&order_by=score desc',
             alert_type=SEARCH_TYPES.RECAP,
         )
         self.assertTrue(
             RECAPPercolator.exists(id=docket_only_alert.pk),
             msg=f"Alert id: {docket_only_alert.pk} was not indexed.",
         )
+        alert_doc = RECAPPercolator.get(id=docket_only_alert.pk)
+        response_str = str(alert_doc.to_dict())
+        self.assertIn("401 Civil", response_str)
+        self.assertIn("'rate': 'wly'", response_str)
+        # function_score breaks percolator queries. Ensure it is never indexed.
+        self.assertNotIn("function_score", response_str)
 
         docket_only_alert_id = docket_only_alert.pk
         # Remove the alert.

cl/api/templates/search-api-docs-vlatest.html

@@ -155,7 +155,7 @@ <h3 id="type">Setting the Result <code>type</code></h3>
     <tbody>
     <tr>
       <td><code>o</code></td>
-      <td>Case law opinions</td>
+      <td>Case law opinion clusters with nested Opinion documents.</td>
     </tr>
     <tr>
       <td><code>r</code></td>

cl/api/templates/webhooks-docs-vlatest.html

@@ -1,6 +1,7 @@
 {% extends "base.html" %}
 {% load static %}
 {% load extras %}
+{% load waffle_tags %}
 
 {% block title %}Webhook API – CourtListener.com{% endblock %}
 {% block description %}
@@ -216,7 +217,7 @@ <h3 id="retries">Retries and Automatic Endpoint Disablement</h3>
   <p>Fixed webhook endpoints can be re-enabled in the webhooks panel.
   </p>
   <p>
-    <img src="{% static "png/re-enable-webhook.png" %}"
+    <img src="{% static "png/re-enable-webhook-v2.png" %}"
        alt="screenshot of how to re-enable a webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -419,6 +420,13 @@ <h2 id="change-log">Change Log</h2>
       <p><strong>v1</strong> First release
       </p>
     </li>
+    <li>
+      <p><strong>v2</strong> - This release introduces support for Case Law Search Alerts results with nested documents.</p>
+      <p>You can now select the webhook version when configuring an endpoint. For most webhook event types, there are no differences between <code>v1</code> and <code>v2</code>, as the payload remains unchanged.
+      </p>
+      <p>In the Search Alert webhook event type, the Oral Arguments search response remains identical between <code>v1</code> and <code>v2</code>.</p>
+      <p>For Case Law {% flag "recap-alerts-active" %}and RECAP{% endflag %} <code>v2</code> now includes nested results, which are based on the new changes introduced in <code>v4</code> of the <a href="{% url "search_api_help" version="v4" %}">Search API.</a></p>
+    </li>
   </ul>
 </div>
 {% endblock %}

cl/api/templates/webhooks-getting-started.html

@@ -104,7 +104,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p>To set up a webhook endpoint, begin by logging into CourtListener and going to the <a href="{% url 'view_webhooks' %}">Webhooks panel in your profile</a>:
   </p>
   <p>
-    <img src="{% static "png/webhooks-panel.png" %}"
+    <img src="{% static "png/webhooks-panel-v2.png" %}"
        alt="screenshot of the webhook panel"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -114,7 +114,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p class="v-offset-above-2">Click the “Add webhook” button and the “Add webhook endpoint” modal pops up:
   </p>
   <p>
-    <img src="{% static "png/add-webhook-endpoint.png" %}"
+    <img src="{% static "png/add-webhook-endpoint-v2.png" %}"
        alt="screenshot of how to add a webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -129,7 +129,11 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
     </li>
     <li>
       <p>Select the Event Type for which you wish to receive events.</p>
-      <p>You can only create one Webhook endpoint for each type of event. Please get in touch if this limitation causes difficulty for your application.
+    </li>
+    <li>
+      <p>Choose the webhook version you wish to set up.</p>
+      <p>We recommend selecting the highest available version for your webhook. Refer to the <a href="{% url "webhooks_docs" version="v2" %}#change-log">Change Log</a> for more details on webhook versions.</p>
+      <p>You can only create one Webhook endpoint for each type of event and version. Please get in touch if this limitation causes difficulty for your application.
       </p>
     </li>
     <li>
@@ -141,7 +145,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p>Click “Create webhook”</p>
   <p>Your Webhook endpoint is now created:</p>
   <p>
-    <img src="{% static "png/webhook-disabled.png" %}"
+    <img src="{% static "png/webhook-disabled-v2.png" %}"
        alt="screenshot of a disabled webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -152,7 +156,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
   <p>Getting a webhook working properly can be difficult, so we have a testing tool that will send you a sample webhook event on demand. </p>
   <p>To use the tool, go to webhooks panel and click the “Test” button for the endpoint you wish to test:</p>
   <p>
-    <img src="{% static "png/webhook-disabled.png" %}"
+    <img src="{% static "png/webhook-disabled-v2.png" %}"
        alt="screenshot of a disabled webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -164,7 +168,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
       <p><strong>In the “As JSON” tab</strong>, you can ask our server to send a test event to your endpoint. When you click “Send Webhook Test Event” a new event is created with the information shown and is sent to your endpoint. Test events are not retried, but can be seen in the “Test Logs” tab.
       </p>
       <p>
-        <img src="{% static "png/test-json-webhook-event.png" %}"
+        <img src="{% static "png/test-json-webhook-event-v2.png" %}"
            alt="screenshot of the webhook json test modal"
            class="img-responsive img-rounded shadow center-block"
            height="261"
@@ -174,7 +178,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
     <li>
       <p><strong>In the “As cURL”</strong> tab, you can copy/paste a curl command that can be used to send a test event to your local dev environment.</p>
       <p>
-        <img src="{% static "png/test-curl-webhook-event.png" %}"
+        <img src="{% static "png/test-curl-webhook-event-v2.png" %}"
            alt="screenshot of the webhook curl test modal"
            class="img-responsive img-rounded shadow center-block"
            height="261"

cl/api/tests.py

@@ -1,8 +1,10 @@
 import json
+from collections import OrderedDict, defaultdict
 from datetime import date, datetime, timedelta, timezone
 from http import HTTPStatus
 from typing import Any, Dict
 from unittest import mock
+from unittest.mock import MagicMock, patch
 from urllib.parse import parse_qs, urlparse
 
 from asgiref.sync import async_to_sync, sync_to_async
@@ -29,7 +31,7 @@
 from cl.api.factories import WebhookEventFactory, WebhookFactory
 from cl.api.models import WEBHOOK_EVENT_STATUS, WebhookEvent, WebhookEventType
 from cl.api.pagination import VersionBasedPagination
-from cl.api.utils import LoggingMixin, get_logging_prefix
+from cl.api.utils import LoggingMixin, get_logging_prefix, invert_user_logs
 from cl.api.views import build_chart_data, coverage_data, make_court_variable
 from cl.api.webhooks import send_webhook_event
 from cl.audio.api_views import AudioViewSet
@@ -3297,3 +3299,159 @@ async def test_count_with_no_results(self):
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data["count"], 0)
+
+
+class TestApiUsage(SimpleTestCase):
+    """Tests for combining v3 and v4 API usage data"""
+
+    def setUp(self):
+        """
+        Set up test environment before each test.
+        We create fresh mocks to avoid state bleeding between tests.
+        """
+        self.mock_redis = MagicMock()
+        self.mock_pipeline = MagicMock()
+        self.mock_redis.pipeline.return_value = self.mock_pipeline
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_single_date_combined_usage(self, mock_get_redis):
+        """
+        Test that for a single date:
+        1. Both v3 and v4 usage is fetched
+        2. Counts are properly combined
+        3. The output format matches template expectations
+        """
+        mock_get_redis.return_value = self.mock_redis
+
+        self.mock_pipeline.execute.return_value = [
+            # First result: v3 API data
+            [("1", 100.0), ("2", 50.0)],
+            # Second result: v4 API data
+            [("1", 50.0), ("2", 25.0)],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-01", add_usernames=False
+        )
+
+        expected = defaultdict(dict)
+        expected[1] = OrderedDict({"2023-01-01": 150, "total": 150})
+        expected[2] = OrderedDict({"2023-01-01": 75, "total": 75})
+
+        self.assertEqual(results, expected)
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_multiple_dates_combined_usage(self, mock_get_redis):
+        """
+        Test that across multiple dates:
+        1. API usage is correctly combined from both v3 and v4
+        2. Totals accumulate properly across dates
+        3. The data structure matches template expectations
+        4. Date ordering is preserved
+        """
+        mock_get_redis.return_value = self.mock_redis
+        self.mock_pipeline.execute.return_value = [
+            # January 1st - v3 API
+            [("1", 100.0), ("2", 50.0)],
+            # January 1st - v4 API
+            [("1", 50.0), ("2", 25.0)],
+            # January 2nd - v3 API
+            [("1", 200.0), ("2", 100.0)],
+            # January 2nd - v4 API
+            [("1", 100.0), ("2", 50.0)],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-02", add_usernames=False
+        )
+
+        # User 1:
+        #   Jan 1: 150 (100 from v3 + 50 from v4)
+        #   Jan 2: 300 (200 from v3 + 100 from v4)
+        #   Total: 450
+        # User 2:
+        #   Jan 1: 75 (50 from v3 + 25 from v4)
+        #   Jan 2: 150 (100 from v3 + 50 from v4)
+        #   Total: 225
+        expected = defaultdict(dict)
+        expected[1] = OrderedDict(
+            {"2023-01-01": 150, "2023-01-02": 300, "total": 450}
+        )
+        expected[2] = OrderedDict(
+            {"2023-01-01": 75, "2023-01-02": 150, "total": 225}
+        )
+
+        self.assertEqual(results, expected)
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_anonymous_user_handling(self, mock_get_redis):
+        """
+        Test the handling of anonymous users, which have special requirements:
+        1. Both 'None' and 'AnonymousUser' should be treated as the same user
+        2. They should be combined under 'AnonymousUser' in the output
+        3. Their usage should be summed correctly across both identifiers
+        4. They should work with both API versions
+        """
+        mock_get_redis.return_value = self.mock_redis
+        self.mock_pipeline.execute.return_value = [
+            # January 1st - v3
+            [
+                ("None", 30.0),
+                ("1", 100.0),
+                ("AnonymousUser", 20.0),
+            ],
+            # January 1st - v4
+            [
+                ("AnonymousUser", 25.0),
+                ("1", 50.0),
+            ],
+            # January 2nd - v3
+            [("None", 40.0), ("1", 200.0)],
+            # January 2nd - v4
+            [
+                ("1", 100.0),
+                ("AnonymousUser", 35.0),
+            ],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-02", add_usernames=False
+        )
+
+        expected = defaultdict(dict)
+        # Expected results:
+        # Anonymous user on Jan 1:
+        #   - v3: 30 (None) + 20 (AnonymousUser) = 50
+        #   - v4: 25 (AnonymousUser) = 25
+        #   - Total for Jan 1: 75
+        # Anonymous user on Jan 2:
+        #   - v3: 40 (None) = 40
+        #   - v4: 35 (AnonymousUser) = 35
+        #   - Total for Jan 2: 75
+        # Anonymous total across all dates: 150
+        expected["AnonymousUser"] = OrderedDict(
+            {
+                "2023-01-01": 75,
+                "2023-01-02": 75,
+                "total": 150,
+            }
+        )
+        expected[1] = OrderedDict(
+            {
+                "2023-01-01": 150,
+                "2023-01-02": 300,
+                "total": 450,
+            }
+        )
+
+        self.assertEqual(results, expected)
+
+        self.assertNotIn("None", results)
+
+        self.assertIn("AnonymousUser", results)
+
+        # Verify ordering is maintained even with anonymous users
+        anonymous_data = results["AnonymousUser"]
+        dates = list(anonymous_data.keys())
+        dates.remove("total")
+        self.assertEqual(dates, ["2023-01-01", "2023-01-02"])