Merge pull request #323 from IATI/develop

Specify timeouts in validate stage
IATI · Mar 20, 2024 · 77305b4 · 77305b4
2 parents d49b495 + a132066
commit 77305b4
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/src/constants/config.py b/src/constants/config.py
@@ -67,12 +67,14 @@
             'SCHEMA_VALIDATION_KEY_NAME'),
         SCHEMA_VALIDATION_KEY_VALUE=os.getenv(
             'SCHEMA_VALIDATION_KEY_VALUE'),
+        SCHEMA_VALIDATION_TIMEOUT=int(os.getenv('SCHEMA_VALIDATOR_API_TIMEOUT', default=3600)),
 
         # Full Validation API URL/key
         FULL_VALIDATION_URL=os.getenv('VALIDATOR_API_URL'),
         FULL_VALIDATION_KEY_NAME=os.getenv('VALIDATOR_API_KEY_NAME'),
         FULL_VALIDATION_KEY_VALUE=os.getenv(
             'VALIDATOR_API_KEY_VALUE'),
+        FULL_VALIDATION_TIMEOUT=int(os.getenv('VALIDATOR_API_TIMEOUT', default=3600)),
 
         # Publisher Black Flagging Period and Threshold
         # Number of Critically Invalid Documents

diff --git a/src/library/validate.py b/src/library/validate.py
@@ -62,7 +62,9 @@ def process_hash_list(document_datasets):
                     f"Schema Validating file hash: {file_hash} and id: {file_id}")
                 schema_headers = {config['VALIDATION']['SCHEMA_VALIDATION_KEY_NAME']: config['VALIDATION']['SCHEMA_VALIDATION_KEY_VALUE']}
                 schema_response = requests.post(
-                    config['VALIDATION']['SCHEMA_VALIDATION_URL'], data=payload.encode('utf-8'), headers=schema_headers)
+                    config['VALIDATION']['SCHEMA_VALIDATION_URL'], data=payload.encode('utf-8'), headers=schema_headers,
+                    timeout=config['VALIDATION']['SCHEMA_VALIDATION_TIMEOUT']
+                )
                 db.updateValidationRequestDate(conn, file_id)
 
                 if schema_response.status_code != 200:
@@ -117,7 +119,8 @@ def process_hash_list(document_datasets):
             if file_schema_valid == False:
                 full_url += '?meta=true'
             full_response = requests.post(
-                full_url, data=payload.encode('utf-8'), headers=full_headers)
+                full_url, data=payload.encode('utf-8'), headers=full_headers,
+                    timeout=config['VALIDATION']['FULL_VALIDATION_TIMEOUT'])
             db.updateValidationRequestDate(conn, file_id)
 
             if full_response.status_code != 200: