factor num stories to meet max stories per page constraint

news_signals/newsapi.py

@@ -150,6 +150,9 @@ def create_newsapi_query(params):
     return dict(template, **{'aql': aql})
 
 
+MAX_STORIES_PER_PAGE = 10 # newsapi constant 2024-07-08
+
+
 def retrieve_stories(params,
                      n_pages=1,
                      headers=HEADERS,
@@ -158,6 +161,17 @@ def retrieve_stories(params,
     params = deepcopy(params)
     stories = []
     cursor = '*'
+    if 'num_stories' in params and params['num_stories'] > MAX_STORIES_PER_PAGE:
+        if n_pages > 1:
+            logger.warning(
+                f"num_stories > {MAX_STORIES_PER_PAGE}, num_stories "
+                f"will be ignored in favor of n_pages"
+            )
+        else:
+            n_pages = params['num_stories'] // MAX_STORIES_PER_PAGE
+            if params['num_stories'] % MAX_STORIES_PER_PAGE:
+                n_pages += 1
+    params['num_stories'] = MAX_STORIES_PER_PAGE
     for i in range(n_pages):
         if verbose:
             logger.info(f'page: {i}, stories: {len(stories)}')

news_signals/test_newsapi.py

@@ -1,7 +1,9 @@
 import os
 import unittest
+from unittest.mock import patch, Mock
 from pathlib import Path
 
+import news_signals
 from news_signals import newsapi
 
 from .log import create_logger
@@ -41,6 +43,42 @@ def test_create_newsapi_query(self):
         with self.assertRaises(TypeError):
             _ = newsapi.create_newsapi_query(params)
 
+    @patch('news_signals.newsapi.make_newsapi_request')
+    def test_num_stories_num_pages(self, mock_make_newsapi_request):
+
+        class MockMakeNewsapiRequest:
+            def __init__(self):
+                self.n_calls = 0
+                self.cursor = 0
+
+            def __call__(self, endpoint, params, headers):
+                self.n_calls += 1
+                self.cursor += 1
+                return {
+                    "stories": [],
+                    "next_page_cursor": self.cursor
+                }
+
+        newsapi_state = MockMakeNewsapiRequest()
+        mock_make_newsapi_request.side_effect = newsapi_state.__call__
+        params = {
+            'entity_surface_forms': ['Tesla'],
+            'categories': ['ay.fin.reports', 'ay.impact.crime'],
+            'num_stories': 100
+        }
+        _ = newsapi.retrieve_stories(params)
+        assert newsapi_state.n_calls == 10
+
+        newsapi_state = MockMakeNewsapiRequest()
+        mock_make_newsapi_request.side_effect = newsapi_state.__call__
+        params = {
+            'entity_surface_forms': ['Tesla'],
+            'categories': ['ay.fin.reports', 'ay.impact.crime'],
+            'num_stories': 10
+        }
+        _ = newsapi.retrieve_stories(params)
+        assert newsapi_state.n_calls == 1
+
 
 class TestResponseValidation(unittest.TestCase):
     @classmethod