From f57335155ad55521f4d3ed010882b6ed1870d28c Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Mon, 18 Dec 2023 10:48:12 -0300 Subject: [PATCH 1/5] Refactored how we get results from Mercadopago. --- config/requirements.txt | 2 +- website/members/management/commands/_mp.py | 34 +++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/config/requirements.txt b/config/requirements.txt index 4448470..72c9eb7 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -9,7 +9,7 @@ django-storages[azure]==1.11.1 django==3.1.14 google-api-python-client==1.12.8 gunicorn==20.0.4 -mercadopago==1.1.1 +mercadopago==2.2.1 oauth2client==4.1.3 Pillow==10.0.1 psycopg2_binary==2.8.6 diff --git a/website/members/management/commands/_mp.py b/website/members/management/commands/_mp.py index 3389599..09fbe6c 100644 --- a/website/members/management/commands/_mp.py +++ b/website/members/management/commands/_mp.py @@ -1,7 +1,7 @@ import logging import os -from mercadopago import MP +from mercadopago import SDK LIMIT = 500 @@ -9,25 +9,37 @@ def get_raw_mercadopago_info(): - """Get records from Mercadopago API.""" - mercadopago_client_id = os.getenv('MERCADOPAGO_CLIENT_ID') - mercadopago_client_secret = os.getenv('MERCADOPAGO_CLIENT_SECRET') + """Get records from Mercadopago API. - mp = MP(mercadopago_client_id, mercadopago_client_secret) + As different hits bring different "total" values indicated, we record what is the + max of those, and only stop hitting when we get no results from one of those endpoints. + """ logger.debug('Connecting with mercadopago') + auth_token = os.getenv('MERCADOPAGO_AUTH_TOKEN') + mp = SDK(auth_token) - filters = {'status': 'approved'} + filters = {'status': 'approved', 'limit': LIMIT, 'begin_date': 'NOW-3MONTHS'} offset = 0 results = [] + max_total_exposed = 0 while True: - response = mp.search_payment(filters, limit=LIMIT, offset=offset) + filters["offset"] = offset + response = mp.payment().search(filters) assert response['status'] == 200 + + quant_results = len(response['response']['results']) + paging = response['response']['paging'] + max_total_exposed = max(max_total_exposed, paging["total"]) logger.debug( - 'Getting response from mercadopago, paging %s', response['response']['paging']) + 'Getting response from mercadopago, len=%d paging=%s', quant_results, paging) + results.extend(response['response']['results']) - if len(response['response']['results']) < LIMIT: - break - offset += LIMIT + if not response['response']['results']: + if paging["total"] == max_total_exposed: + # didn't get any result when hitting the endpoint that brings more + # results that any other + break + offset += quant_results logger.info('Got response from mercadopago, %d items', len(results)) return results From 38a765e2fd4e0f7aff0e7309bff769e696672d7b Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Mon, 18 Dec 2023 14:00:36 -0300 Subject: [PATCH 2/5] Fixed docker compose version. --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 40b8c94..f52d192 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: python-version: 3.9 - name: Bootstrap everything run: | - pip install -U docker-compose + pip install -U "docker-compose < 7" docker-compose up -d docker-compose exec -T web pip install -r /code/config/requirements-dev.txt - name: Run tests From d6916b5d81f274885b9b4b62fff3e6e47e7c5956 Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Mon, 18 Dec 2023 14:04:40 -0300 Subject: [PATCH 3/5] Fixed *docker* version. --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f52d192..14daa54 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,8 @@ jobs: python-version: 3.9 - name: Bootstrap everything run: | - pip install -U "docker-compose < 7" + pip install -U docker < 7 # compatibility issues; see https://github.com/docker/docker-py/issues/3194 + pip install -U docker-compose docker-compose up -d docker-compose exec -T web pip install -r /code/config/requirements-dev.txt - name: Run tests From 02ecd455096492642f1e21fe2fcd8a35b6cc2a55 Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Mon, 18 Dec 2023 14:05:52 -0300 Subject: [PATCH 4/5] Quotes, you silly --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 14daa54..a377df3 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: python-version: 3.9 - name: Bootstrap everything run: | - pip install -U docker < 7 # compatibility issues; see https://github.com/docker/docker-py/issues/3194 + pip install -U "docker < 7" # compatibility issues; see https://github.com/docker/docker-py/issues/3194 pip install -U docker-compose docker-compose up -d docker-compose exec -T web pip install -r /code/config/requirements-dev.txt From 72d91e4b941f43197f6bbbfb99855cf626319440 Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Tue, 19 Dec 2023 10:43:45 -0300 Subject: [PATCH 5/5] Better comment wording. --- website/members/management/commands/_mp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/members/management/commands/_mp.py b/website/members/management/commands/_mp.py index 09fbe6c..b3d6766 100644 --- a/website/members/management/commands/_mp.py +++ b/website/members/management/commands/_mp.py @@ -36,8 +36,7 @@ def get_raw_mercadopago_info(): results.extend(response['response']['results']) if not response['response']['results']: if paging["total"] == max_total_exposed: - # didn't get any result when hitting the endpoint that brings more - # results that any other + # didn't get any result and reported "total" is the biggest seen so far break offset += quant_results