From 43a12eb828fd5277a92aae84dd88105bb6f47ed2 Mon Sep 17 00:00:00 2001 From: Ruby Salton Date: Tue, 2 Nov 2021 18:31:59 +0200 Subject: [PATCH] fix SEC error 403: add user-agent header to EDGAR request to comply with the "Fair access" guidelines in https://www.sec.gov/os/accessing-edgar-data: "The SEC does not allow botnets or automated tools to crawl the site. Any request that has been identified as part of a botnet or an automated tool outside of the acceptable policy will be managed to ensure fair access for all users. Please declare your user agent in request headers" --- edgar/client.py | 4 ++-- edgar/session.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/edgar/client.py b/edgar/client.py index 2787bbe..d3a83fd 100644 --- a/edgar/client.py +++ b/edgar/client.py @@ -22,7 +22,7 @@ class EdgarClient(): instantiate the different endpoints. """ - def __init__(self) -> None: + def __init__(self, user_agent: str) -> None: """Initializes the `EdgarClient`. ### Usage @@ -30,7 +30,7 @@ def __init__(self) -> None: >>> edgar_client = EdgarClient() """ - self.edgar_session = EdgarSession(client=self) + self.edgar_session = EdgarSession(client=self, user_agent=user_agent) def __repr__(self) -> str: """String representation of the `EdgarClient` object.""" diff --git a/edgar/session.py b/edgar/session.py index 6e3c37c..79ece58 100644 --- a/edgar/session.py +++ b/edgar/session.py @@ -18,7 +18,7 @@ class EdgarSession(): handles all the requests made to EDGAR. """ - def __init__(self, client: object) -> None: + def __init__(self, client: object, user_agent: str) -> None: """Initializes the `EdgarSession` client. ### Parameters: @@ -40,6 +40,7 @@ def __init__(self, client: object) -> None: self.resource = 'https://www.sec.gov' self.api_resource = 'https://data.sec.gov' self.total_requests = 0 + self.user_agent = user_agent if not pathlib.Path('logs').exists(): pathlib.Path('logs').mkdir() @@ -151,6 +152,7 @@ def make_request( # Define a new request. request_request = requests.Request( + headers={'user-agent': self.user_agent}, method=method.upper(), url=url, params=params,