Merge branch 'main' into 4826-replicate-pdf-uploads-to-subdockets

cl/api/templates/search-api-docs-vlatest.html

@@ -154,7 +154,7 @@ <h3 id="type">Setting the Result <code>type</code></h3>
     <tbody>
     <tr>
       <td><code>o</code></td>
-      <td>Case law opinions</td>
+      <td>Case law opinion clusters with nested Opinion documents.</td>
     </tr>
     <tr>
       <td><code>r</code></td>

cl/api/templates/webhooks-docs-vlatest.html

@@ -1,6 +1,7 @@
 {% extends "base.html" %}
 {% load static %}
 {% load extras %}
+{% load waffle_tags %}
 
 {% block title %}Webhook API – CourtListener.com{% endblock %}
 {% block description %}
@@ -216,7 +217,7 @@ <h3 id="retries">Retries and Automatic Endpoint Disablement</h3>
   <p>Fixed webhook endpoints can be re-enabled in the webhooks panel.
   </p>
   <p>
-    <img src="{% static "png/re-enable-webhook.png" %}"
+    <img src="{% static "png/re-enable-webhook-v2.png" %}"
        alt="screenshot of how to re-enable a webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -419,6 +420,13 @@ <h2 id="change-log">Change Log</h2>
       <p><strong>v1</strong> First release
       </p>
     </li>
+    <li>
+      <p><strong>v2</strong> - This release introduces support for Case Law Search Alerts results with nested documents.</p>
+      <p>You can now select the webhook version when configuring an endpoint. For most webhook event types, there are no differences between <code>v1</code> and <code>v2</code>, as the payload remains unchanged.
+      </p>
+      <p>In the Search Alert webhook event type, the Oral Arguments search response remains identical between <code>v1</code> and <code>v2</code>.</p>
+      <p>For Case Law {% flag "recap-alerts-active" %}and RECAP{% endflag %} <code>v2</code> now includes nested results, which are based on the new changes introduced in <code>v4</code> of the <a href="{% url "search_api_help" version="v4" %}">Search API.</a></p>
+    </li>
   </ul>
 </div>
 {% endblock %}

cl/api/templates/webhooks-getting-started.html

@@ -104,7 +104,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p>To set up a webhook endpoint, begin by logging into CourtListener and going to the <a href="{% url 'view_webhooks' %}">Webhooks panel in your profile</a>:
   </p>
   <p>
-    <img src="{% static "png/webhooks-panel.png" %}"
+    <img src="{% static "png/webhooks-panel-v2.png" %}"
        alt="screenshot of the webhook panel"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -114,7 +114,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p class="v-offset-above-2">Click the “Add webhook” button and the “Add webhook endpoint” modal pops up:
   </p>
   <p>
-    <img src="{% static "png/add-webhook-endpoint.png" %}"
+    <img src="{% static "png/add-webhook-endpoint-v2.png" %}"
        alt="screenshot of how to add a webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -129,7 +129,11 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
     </li>
     <li>
       <p>Select the Event Type for which you wish to receive events.</p>
-      <p>You can only create one Webhook endpoint for each type of event. Please get in touch if this limitation causes difficulty for your application.
+    </li>
+    <li>
+      <p>Choose the webhook version you wish to set up.</p>
+      <p>We recommend selecting the highest available version for your webhook. Refer to the <a href="{% url "webhooks_docs" version="v2" %}#change-log">Change Log</a> for more details on webhook versions.</p>
+      <p>You can only create one Webhook endpoint for each type of event and version. Please get in touch if this limitation causes difficulty for your application.
       </p>
     </li>
     <li>
@@ -141,7 +145,7 @@ <h2 id="setup-a-webhook">Set Up a Webhook Endpoint in CourtListener</h2>
   <p>Click “Create webhook”</p>
   <p>Your Webhook endpoint is now created:</p>
   <p>
-    <img src="{% static "png/webhook-disabled.png" %}"
+    <img src="{% static "png/webhook-disabled-v2.png" %}"
        alt="screenshot of a disabled webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -152,7 +156,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
   <p>Getting a webhook working properly can be difficult, so we have a testing tool that will send you a sample webhook event on demand. </p>
   <p>To use the tool, go to webhooks panel and click the “Test” button for the endpoint you wish to test:</p>
   <p>
-    <img src="{% static "png/webhook-disabled.png" %}"
+    <img src="{% static "png/webhook-disabled-v2.png" %}"
        alt="screenshot of a disabled webhook endpoint"
        class="img-responsive img-rounded shadow center-block"
        height="261"
@@ -164,7 +168,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
       <p><strong>In the “As JSON” tab</strong>, you can ask our server to send a test event to your endpoint. When you click “Send Webhook Test Event” a new event is created with the information shown and is sent to your endpoint. Test events are not retried, but can be seen in the “Test Logs” tab.
       </p>
       <p>
-        <img src="{% static "png/test-json-webhook-event.png" %}"
+        <img src="{% static "png/test-json-webhook-event-v2.png" %}"
            alt="screenshot of the webhook json test modal"
            class="img-responsive img-rounded shadow center-block"
            height="261"
@@ -174,7 +178,7 @@ <h2 id="testing">Testing a Webhook endpoint.</h2>
     <li>
       <p><strong>In the “As cURL”</strong> tab, you can copy/paste a curl command that can be used to send a test event to your local dev environment.</p>
       <p>
-        <img src="{% static "png/test-curl-webhook-event.png" %}"
+        <img src="{% static "png/test-curl-webhook-event-v2.png" %}"
            alt="screenshot of the webhook curl test modal"
            class="img-responsive img-rounded shadow center-block"
            height="261"

cl/api/tests.py

@@ -1,8 +1,10 @@
 import json
+from collections import OrderedDict, defaultdict
 from datetime import date, datetime, timedelta, timezone
 from http import HTTPStatus
 from typing import Any, Dict
 from unittest import mock
+from unittest.mock import MagicMock, patch
 from urllib.parse import parse_qs, urlparse
 
 from asgiref.sync import async_to_sync, sync_to_async
@@ -29,7 +31,7 @@
 from cl.api.factories import WebhookEventFactory, WebhookFactory
 from cl.api.models import WEBHOOK_EVENT_STATUS, WebhookEvent, WebhookEventType
 from cl.api.pagination import VersionBasedPagination
-from cl.api.utils import LoggingMixin, get_logging_prefix
+from cl.api.utils import LoggingMixin, get_logging_prefix, invert_user_logs
 from cl.api.views import build_chart_data, coverage_data, make_court_variable
 from cl.api.webhooks import send_webhook_event
 from cl.audio.api_views import AudioViewSet
@@ -3297,3 +3299,159 @@ async def test_count_with_no_results(self):
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data["count"], 0)
+
+
+class TestApiUsage(SimpleTestCase):
+    """Tests for combining v3 and v4 API usage data"""
+
+    def setUp(self):
+        """
+        Set up test environment before each test.
+        We create fresh mocks to avoid state bleeding between tests.
+        """
+        self.mock_redis = MagicMock()
+        self.mock_pipeline = MagicMock()
+        self.mock_redis.pipeline.return_value = self.mock_pipeline
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_single_date_combined_usage(self, mock_get_redis):
+        """
+        Test that for a single date:
+        1. Both v3 and v4 usage is fetched
+        2. Counts are properly combined
+        3. The output format matches template expectations
+        """
+        mock_get_redis.return_value = self.mock_redis
+
+        self.mock_pipeline.execute.return_value = [
+            # First result: v3 API data
+            [("1", 100.0), ("2", 50.0)],
+            # Second result: v4 API data
+            [("1", 50.0), ("2", 25.0)],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-01", add_usernames=False
+        )
+
+        expected = defaultdict(dict)
+        expected[1] = OrderedDict({"2023-01-01": 150, "total": 150})
+        expected[2] = OrderedDict({"2023-01-01": 75, "total": 75})
+
+        self.assertEqual(results, expected)
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_multiple_dates_combined_usage(self, mock_get_redis):
+        """
+        Test that across multiple dates:
+        1. API usage is correctly combined from both v3 and v4
+        2. Totals accumulate properly across dates
+        3. The data structure matches template expectations
+        4. Date ordering is preserved
+        """
+        mock_get_redis.return_value = self.mock_redis
+        self.mock_pipeline.execute.return_value = [
+            # January 1st - v3 API
+            [("1", 100.0), ("2", 50.0)],
+            # January 1st - v4 API
+            [("1", 50.0), ("2", 25.0)],
+            # January 2nd - v3 API
+            [("1", 200.0), ("2", 100.0)],
+            # January 2nd - v4 API
+            [("1", 100.0), ("2", 50.0)],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-02", add_usernames=False
+        )
+
+        # User 1:
+        #   Jan 1: 150 (100 from v3 + 50 from v4)
+        #   Jan 2: 300 (200 from v3 + 100 from v4)
+        #   Total: 450
+        # User 2:
+        #   Jan 1: 75 (50 from v3 + 25 from v4)
+        #   Jan 2: 150 (100 from v3 + 50 from v4)
+        #   Total: 225
+        expected = defaultdict(dict)
+        expected[1] = OrderedDict(
+            {"2023-01-01": 150, "2023-01-02": 300, "total": 450}
+        )
+        expected[2] = OrderedDict(
+            {"2023-01-01": 75, "2023-01-02": 150, "total": 225}
+        )
+
+        self.assertEqual(results, expected)
+
+    @patch("cl.api.utils.get_redis_interface")
+    def test_anonymous_user_handling(self, mock_get_redis):
+        """
+        Test the handling of anonymous users, which have special requirements:
+        1. Both 'None' and 'AnonymousUser' should be treated as the same user
+        2. They should be combined under 'AnonymousUser' in the output
+        3. Their usage should be summed correctly across both identifiers
+        4. They should work with both API versions
+        """
+        mock_get_redis.return_value = self.mock_redis
+        self.mock_pipeline.execute.return_value = [
+            # January 1st - v3
+            [
+                ("None", 30.0),
+                ("1", 100.0),
+                ("AnonymousUser", 20.0),
+            ],
+            # January 1st - v4
+            [
+                ("AnonymousUser", 25.0),
+                ("1", 50.0),
+            ],
+            # January 2nd - v3
+            [("None", 40.0), ("1", 200.0)],
+            # January 2nd - v4
+            [
+                ("1", 100.0),
+                ("AnonymousUser", 35.0),
+            ],
+        ]
+
+        results = invert_user_logs(
+            start="2023-01-01", end="2023-01-02", add_usernames=False
+        )
+
+        expected = defaultdict(dict)
+        # Expected results:
+        # Anonymous user on Jan 1:
+        #   - v3: 30 (None) + 20 (AnonymousUser) = 50
+        #   - v4: 25 (AnonymousUser) = 25
+        #   - Total for Jan 1: 75
+        # Anonymous user on Jan 2:
+        #   - v3: 40 (None) = 40
+        #   - v4: 35 (AnonymousUser) = 35
+        #   - Total for Jan 2: 75
+        # Anonymous total across all dates: 150
+        expected["AnonymousUser"] = OrderedDict(
+            {
+                "2023-01-01": 75,
+                "2023-01-02": 75,
+                "total": 150,
+            }
+        )
+        expected[1] = OrderedDict(
+            {
+                "2023-01-01": 150,
+                "2023-01-02": 300,
+                "total": 450,
+            }
+        )
+
+        self.assertEqual(results, expected)
+
+        self.assertNotIn("None", results)
+
+        self.assertIn("AnonymousUser", results)
+
+        # Verify ordering is maintained even with anonymous users
+        anonymous_data = results["AnonymousUser"]
+        dates = list(anonymous_data.keys())
+        dates.remove("total")
+        self.assertEqual(dates, ["2023-01-01", "2023-01-02"])

cl/api/utils.py

@@ -1,6 +1,7 @@
 import logging
 from collections import OrderedDict, defaultdict
 from datetime import date, datetime, timedelta, timezone
+from itertools import batched, chain
 from typing import Any, Dict, List, Set, TypedDict, Union
 
 import eyecite
@@ -704,56 +705,64 @@ def invert_user_logs(
     end: Union[str, datetime],
     add_usernames: bool = True,
 ) -> Dict[str, Dict[str, int]]:
-    """Invert the user logs for a period of time
+    """Aggregate API usage statistics per user over a date range.
 
-    The user logs have the date in the key and the user as part of the set:
+    - Anonymous users are aggregated under the key 'AnonymousUser'.
+    - Both v3 and v4 API counts are combined in the results.
 
-        'api:v3.user.d:2016-10-01.counts': {
-           mlissner: 22,
-           joe_hazard: 33,
-        }
-
-    This inverts these entries to:
+    :param start: Beginning date (inclusive) for the query range
+    :param end: End date (inclusive) for the query range
+    :param add_usernames: If True, replaces user IDs with usernames as keys.
+        When False, uses only user IDs as keys.
 
-        users: {
-            mlissner: {
-                2016-10-01: 22,
-                total: 22,
-            },
-            joe_hazard: {
-                2016-10-01: 33,
-                total: 33,
-            }
-        }
-    :param start: The beginning date (inclusive) you want the results for. A
-    :param end: The end date (inclusive) you want the results for.
-    :param add_usernames: Stats are stored with the user ID. If this is True,
-    add an alias in the returned dictionary that contains the username as well.
-    :return The inverted dictionary
+    :return: Dictionary mapping user identifiers (usernames if `add_usernames=True`,
+        otherwise user IDs) to their daily API usage counts and totals.
+        Inner dictionaries are ordered by date. Only dates with usage are included.
     """
     r = get_redis_interface("STATS")
     pipe = r.pipeline()
 
     dates = make_date_str_list(start, end)
+    versions = ["v3", "v4"]
     for d in dates:
-        pipe.zrange(f"api:v3.user.d:{d}.counts", 0, -1, withscores=True)
+        for version in versions:
+            pipe.zrange(
+                f"api:{version}.user.d:{d}.counts",
+                0,
+                -1,
+                withscores=True,
+            )
+
+    # results contains alternating v3/v4 API usage data for each date queried.
+    # For example, if querying 2023-01-01 to 2023-01-02, results might look like:
+    # [
+    #     # 2023-01-01 v3 data: [(user_id, count), ...]
+    #     [("1", 100.0), ("2", 50.0)],
+    #     # 2023-01-01 v4 data
+    #     [("1", 50.0), ("2", 25.0)],
+    #     # 2023-01-02 v3 data
+    #     [("1", 200.0), ("2", 100.0)],
+    #     # 2023-01-02 v4 data
+    #     [("1", 100.0), ("2", 50.0)]
+    # ]
+    # We zip this with dates to combine v3/v4 counts per user per day
     results = pipe.execute()
 
-    # results is a list of results for each of the zrange queries above. Zip
-    # those results with the date that created it, and invert the whole thing.
     out: defaultdict = defaultdict(dict)
-    for d, result in zip(dates, results):
-        for user_id, count in result:
-            if user_id == "None" or user_id == "AnonymousUser":
-                user_id = "AnonymousUser"
-            else:
-                user_id = int(user_id)
-            count = int(count)
-            if out.get(user_id):
-                out[user_id][d] = count
-                out[user_id]["total"] += count
-            else:
-                out[user_id] = {d: count, "total": count}
+
+    def update_user_counts(_user_id, _count, _date):
+        user_is_anonymous = _user_id == "None" or _user_id == "AnonymousUser"
+        _user_id = "AnonymousUser" if user_is_anonymous else int(_user_id)
+        _count = int(_count)
+        out.setdefault(_user_id, OrderedDict())
+        out[_user_id].setdefault(_date, 0)
+        out[_user_id][_date] += _count
+        out[_user_id].setdefault("total", 0)
+        out[_user_id]["total"] += _count
+
+    for d, api_usage in zip(dates, batched(results, len(versions))):
+        for user_id, count in chain(*api_usage):
+            update_user_counts(user_id, count, d)
 
     # Sort the values
     for k, v in out.items():