Skip to content

Commit

Permalink
Ignore coles errors since some things seem to be broken
Browse files Browse the repository at this point in the history
  • Loading branch information
Javex committed Nov 3, 2024
1 parent af83327 commit 4593ac9
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions hotprices_au/sites/coles.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# How many errors are acceptable for a category before failing?
# This means it's okay if *some* categories don't return 100% of products
ERROR_COUNT_MAX = 9
ERROR_IGNORE = True


class ColesScraper:
Expand Down Expand Up @@ -60,10 +61,11 @@ def get_category(self, cat_slug, page_filter: int):
except requests.HTTPError:
error_count += 1
print(f'Error fetching page {params["page"]}')
# Need to also raise an error if there's a page filter as there
# are no more pages to try
if error_count > ERROR_COUNT_MAX or page_filter is not None:
raise
if not ERROR_IGNORE:
# Need to also raise an error if there's a page filter as there
# are no more pages to try
if error_count > ERROR_COUNT_MAX or page_filter is not None:
raise
else:
params["page"] += 1
continue
Expand Down

0 comments on commit 4593ac9

Please sign in to comment.