Add groups query to the bulk API

h/routes.py

@@ -131,6 +131,7 @@ def includeme(config):  # pylint: disable=too-many-statements
     config.add_route(
         "api.bulk.annotation", "/api/bulk/annotation", request_method="POST"
     )
+    config.add_route("api.bulk.group", "/api/bulk/group", request_method="POST")
 
     config.add_route("api.groups", "/api/groups", factory="h.traversal.GroupRoot")
     config.add_route(

h/security/policy/_basic_http_auth.py

@@ -39,6 +39,7 @@ class AuthClientPolicy(IdentityBasedPolicy):
         ("api.user", "PATCH"),
         ("api.bulk.action", "POST"),
         ("api.bulk.annotation", "POST"),
+        ("api.bulk.group", "POST"),
     ]
 
     @classmethod

h/services/__init__.py

@@ -3,7 +3,7 @@
 from h.services.annotation_read import AnnotationReadService
 from h.services.annotation_write import AnnotationWriteService
 from h.services.auth_cookie import AuthCookieService
-from h.services.bulk_annotation import BulkAnnotationService
+from h.services.bulk_api import BulkAnnotationService, BulkGroupService
 from h.services.subscription import SubscriptionService
 
 
@@ -41,7 +41,10 @@ def includeme(config):  # pragma: no cover
         "h.services.auth_token.auth_token_service_factory", name="auth_token"
     )
     config.register_service_factory(
-        "h.services.bulk_annotation.service_factory", iface=BulkAnnotationService
+        "h.services.bulk_api.annotation.service_factory", iface=BulkAnnotationService
+    )
+    config.register_service_factory(
+        "h.services.bulk_api.group.service_factory", iface=BulkGroupService
     )
     config.register_service_factory(
         "h.services.developer_token.developer_token_service_factory",

h/services/bulk_api/__init__.py

@@ -0,0 +1,3 @@
+from h.services.bulk_api.annotation import BulkAnnotation, BulkAnnotationService
+from h.services.bulk_api.core import BadDateFilter
+from h.services.bulk_api.group import BulkGroupService

h/services/bulk_annotation.py → h/services/bulk_api/annotation.py

@@ -6,52 +6,7 @@
 from sqlalchemy.sql import Select
 
 from h.models import AnnotationMetadata, AnnotationSlim, Group, GroupMembership, User
-
-
-class BadDateFilter(Exception):
-    """There is something wrong with the date filter provided."""
-
-
-def date_match(column: sa.Column, spec: dict):
-    """
-    Get an SQL comparator for a date column based on dict spec.
-
-    The dict can contain operators as keys and dates as values as per the
-    following complete (but nonsensical) filter:
-
-        {
-            "gt": "2012-11-30",
-            "gte": "2012-11-30",
-            "lt": "2012-11-30",
-            "lte": "2012-11-30",
-            "eq": "2012-11-30",
-            "ne": "2012-11-30",
-        }
-
-    :raises BadDateFilter: For unrecognised operators or no spec
-    """
-    if not spec:
-        raise BadDateFilter(f"No spec given to filter '{column}' on")
-
-    clauses = []
-
-    for op_key, value in spec.items():
-        if op_key == "gt":
-            clauses.append(column > value)
-        elif op_key == "gte":
-            clauses.append(column >= value)
-        elif op_key == "lt":
-            clauses.append(column < value)
-        elif op_key == "lte":
-            clauses.append(column <= value)
-        elif op_key == "eq":
-            clauses.append(column == value)
-        elif op_key == "ne":
-            clauses.append(column != value)
-        else:
-            raise BadDateFilter(f"Unknown date filter operator: {op_key}")
-
-    return sa.and_(*clauses)
+from h.services.bulk_api.core import date_match
 
 
 @dataclass

h/services/bulk_api/core.py

@@ -0,0 +1,47 @@
+import sqlalchemy as sa
+
+
+class BadDateFilter(Exception):
+    """There is something wrong with the date filter provided."""
+
+
+def date_match(column: sa.Column, spec: dict):
+    """
+    Get an SQL comparator for a date column based on dict spec.
+
+    The dict can contain operators as keys and dates as values as per the
+    following complete (but nonsensical) filter:
+
+        {
+            "gt": "2012-11-30",
+            "gte": "2012-11-30",
+            "lt": "2012-11-30",
+            "lte": "2012-11-30",
+            "eq": "2012-11-30",
+            "ne": "2012-11-30",
+        }
+
+    :raises BadDateFilter: For unrecognised operators or no spec
+    """
+    if not spec:
+        raise BadDateFilter(f"No spec given to filter '{column}' on")
+
+    clauses = []
+
+    for op_key, value in spec.items():
+        if op_key == "gt":
+            clauses.append(column > value)
+        elif op_key == "gte":
+            clauses.append(column >= value)
+        elif op_key == "lt":
+            clauses.append(column < value)
+        elif op_key == "lte":
+            clauses.append(column <= value)
+        elif op_key == "eq":
+            clauses.append(column == value)
+        elif op_key == "ne":
+            clauses.append(column != value)
+        else:
+            raise BadDateFilter(f"Unknown date filter operator: {op_key}")
+
+    return sa.and_(*clauses)

h/services/bulk_api/group.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from typing import List
+
+import sqlalchemy as sa
+from sqlalchemy.orm import Session
+
+from h.models import Annotation, Group
+from h.services.bulk_api.core import date_match
+
+
+@dataclass
+class BulkGroup:
+    authority_provided_id: str
+
+
+class BulkGroupService:
+    """A service for retrieving groups in bulk."""
+
+    def __init__(self, db: Session):
+        self._db = db
+
+    def group_search(
+        self, groups: List[str], annotations_created: dict
+    ) -> List[BulkGroup]:
+        query = (
+            sa.select([Group.authority_provided_id])
+            .join(Annotation, Group.pubid == Annotation.groupid)
+            .group_by(Group.authority_provided_id)
+            .where(
+                date_match(Annotation.created, annotations_created),
+                Group.authority_provided_id.in_(groups),
+            )
+        )
+        results = self._db.scalars(query)
+        return [BulkGroup(authority_provided_id=row) for row in results.all()]
+
+
+def service_factory(_context, request) -> BulkGroupService:
+    return BulkGroupService(db=request.db)

h/views/api/bulk/annotation.py

@@ -6,7 +6,7 @@
 from h.schemas.base import JSONSchema
 from h.security import Permission
 from h.services import BulkAnnotationService
-from h.services.bulk_annotation import BadDateFilter, BulkAnnotation
+from h.services.bulk_api import BadDateFilter, BulkAnnotation
 from h.views.api.bulk._ndjson import get_ndjson_response
 from h.views.api.config import api_config
 

h/views/api/bulk/group.py

@@ -0,0 +1,44 @@
+import json
+
+from importlib_resources import files
+
+from h.schemas import ValidationError
+from h.schemas.base import JSONSchema
+from h.security import Permission
+from h.services.bulk_api import BadDateFilter, BulkGroupService
+from h.views.api.bulk._ndjson import get_ndjson_response
+from h.views.api.config import api_config
+
+
+class BulkGroupSchema(JSONSchema):
+    _SCHEMA_FILE = files("h.views.api.bulk") / "group_schema.json"
+
+    schema_version = 7
+    schema = json.loads(_SCHEMA_FILE.read_text(encoding="utf-8"))
+
+
+@api_config(
+    versions=["v1", "v2"],
+    route_name="api.bulk.group",
+    request_method="POST",
+    link_name="bulk.group",
+    description="Retrieve a large number of groups in one go",
+    subtype="x-ndjson",
+    permission=Permission.API.BULK_ACTION,
+)
+def bulk_group(request):
+    data = BulkGroupSchema().validate(request.json)
+    query_filter = data["filter"]
+
+    try:
+        groups = request.find_service(BulkGroupService).group_search(
+            groups=query_filter["groups"],
+            annotations_created=query_filter["annotations_created"],
+        )
+
+    except BadDateFilter as err:
+        raise ValidationError(str(err)) from err
+
+    return get_ndjson_response(
+        [{"authority_provided_id": group.authority_provided_id} for group in groups]
+    )

h/views/api/bulk/group_schema.json

@@ -0,0 +1,52 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "type": "object",
+    "title": "Bulk Group Request",
+    "examples": [
+        {
+            "filter": {
+                "groups": ["3a022b6c146dfd9df4ea8662178eac"],
+                "annotations_created": {
+                    "gt": "2018-11-13T20:20:39+00:00",
+                    "lte": "2018-11-13T20:20:39+00:00"
+                }
+            }
+        }
+    ],
+    "properties": {
+        "filter": {"$ref": "#/$defs/filter"}
+    },
+    "required": ["filter"],
+    "additionalProperties": true,
+    "$defs": {
+        "filter": {
+            "title": "Filter query",
+            "description": "The filters to search for the annotations by",
+
+            "type": "object",
+            "properties": {
+                "groups": {"$ref": "#/$defs/groupsFilter"},
+                "annotations_created": {"$ref": "#/$defs/dateFilter"}
+            },
+            "required": ["groups", "annotations_created"],
+            "additionalProperties": false
+        },
+
+        "groupsFilter": {
+                "type": "array",
+                "minItems": 1,
+                "items": {"type": "string"}
+        },
+        "dateFilter": {
+            "description": "A filter to apply on a date",
+
+            "type": "object",
+            "properties": {
+                "gt": {"type": "string", "format": "date-time"},
+                "lte": {"type": "string", "format": "date-time"}
+            },
+            "required": ["gt", "lte"],
+            "additionalProperties": false
+        }
+    }
+}

tests/common/fixtures/services.py

@@ -10,7 +10,7 @@
 from h.services.annotation_write import AnnotationWriteService
 from h.services.auth_cookie import AuthCookieService
 from h.services.auth_token import AuthTokenService
-from h.services.bulk_annotation import BulkAnnotationService
+from h.services.bulk_api import BulkAnnotationService, BulkGroupService
 from h.services.flag import FlagService
 from h.services.group import GroupService
 from h.services.group_create import GroupCreateService
@@ -45,6 +45,7 @@
     "auth_cookie_service",
     "auth_token_service",
     "bulk_annotation_service",
+    "bulk_group_service",
     "links_service",
     "list_organizations_service",
     "flag_service",
@@ -128,6 +129,11 @@ def bulk_annotation_service(mock_service):
     return mock_service(BulkAnnotationService)
 
 
+@pytest.fixture
+def bulk_group_service(mock_service):
+    return mock_service(BulkGroupService)
+
+
 @pytest.fixture
 def links_service(mock_service):
     return mock_service(LinksService, name="links")

tests/unit/h/routes_test.py

@@ -124,6 +124,7 @@ def test_includeme():
         ),
         call("api.bulk.action", "/api/bulk", request_method="POST"),
         call("api.bulk.annotation", "/api/bulk/annotation", request_method="POST"),
+        call("api.bulk.group", "/api/bulk/group", request_method="POST"),
         call("api.groups", "/api/groups", factory="h.traversal.GroupRoot"),
         call(
             "api.group_upsert",