diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aa3f0dc..d834e25 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.12 + python-version: 3.9 cache: "pip" cache-dependency-path: "**/requirements_dev.txt" - name: Run tests @@ -37,5 +37,6 @@ jobs: export SETTINGS_DIR=. pip install -r requirements_dev.txt pip install -e . - # pre-commit run --all-files TODO: check what's wrong in CI + pre-commit run --all-files pytest fastapi_tests + pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66a8711..c306597 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,18 @@ -exclude: ^(data/|app/templates/|tests/data) repos: - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: - - --profile=black - - --filter-files - - --force-grid-wrap=3 - - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.8.0 hooks: - id: black language_version: python3 - - repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 hooks: - id: flake8 language_version: python3 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace diff --git a/.travis.yml b/.travis.yml index b08e9e5..9f44f39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,6 @@ install: before_script: - psql -c 'create database defaultdb;' -U postgres - #- hug -f tests/service.py & - - gunicorn tests.service:__hug_wsgi__ -D - - sleep 2 script: tox diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b31e771..66bc0f2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ History ======= +0.96.2 (2024-10-07) +------------------- +* Removed deprecated peewee autorollback option. + 0.96.1 (2024-09-26) ------------------- * Added annotated fastapi dependencies. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6c13b9b..260d134 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -83,7 +83,6 @@ Ready to contribute? Here's how to set up `apphelpers` for local development. $ export SETTINGS_DIR=. $ docker-compose up -d # start postgres and redis - $ gunicorn tests.service:__hug_wsgi__ $ pytest tests 6. When you're done making changes for fastapi, check that your changes pass flake8 and the @@ -92,7 +91,6 @@ Ready to contribute? Here's how to set up `apphelpers` for local development. $ export SETTINGS_DIR=. $ docker-compose up -d # start postgres and redis - $ uvicorn fastapi_tests.service:app --host 0.0.0.0 --port 5000 $ pytest fastapi_tests 7. Commit your changes and push your branch to GitHub:: diff --git a/apphelpers/__init__.py b/apphelpers/__init__.py index f370591..c1230b5 100644 --- a/apphelpers/__init__.py +++ b/apphelpers/__init__.py @@ -4,4 +4,4 @@ __author__ = """Scroll Tech""" __email__ = "engg@stck.me" -__version__ = "0.96.1" +__version__ = "0.96.2" diff --git a/apphelpers/db/peewee.py b/apphelpers/db/peewee.py index bd5e581..93ddc91 100644 --- a/apphelpers/db/peewee.py +++ b/apphelpers/db/peewee.py @@ -41,7 +41,6 @@ def create_pgdb_pool( user=user, password=password, max_connections=max_connections, - autorollback=True, register_hstore=False, stale_timeout=60 * 2, ) # 2 minutes @@ -54,7 +53,12 @@ class Meta: only_save_dirty = True def to_dict(self, only=None, exclude=None, recurse=False): - return model_to_dict(self, only=only, exclude=exclude, recurse=recurse) + return model_to_dict( + self, + only=only, + exclude=exclude, + recurse=recurse, + ) return BaseModel @@ -77,8 +81,8 @@ def dbtransaction_ctx(db): def dbtransaction(db): """ wrapper that make db transactions automic - note db connections are used only when it is needed (hence there is no usual - connection open/close) + note db connections are used only when it is needed (hence there is no + usual connection open/close) """ def wrapper(f): @@ -97,8 +101,8 @@ def enumify(TheModel, name_field="name", val_field="id"): Converts a model rows into an enum Can be effective cache for mostly unchanging data. Limitation: No auto updates. If you update the model and you are using - process manager like gunicorn you would need to restart to rnsure enums are - updated + process manager like gunicorn you would need to restart to rnsure enums + are updated eg. >>> class Week(BaseModel): @@ -130,7 +134,8 @@ def get_sub_models(base_model): # Useful functions for test/dev setups -# NOTE: For below functions, models is list of model classes sorted by dependency +# NOTE: For below functions, models is list of model classes sorted by +# dependency # Example: [Author, Publication, Post, Comment] diff --git a/apphelpers/db/piccolo.py b/apphelpers/db/piccolo.py index e5f6904..d88014f 100644 --- a/apphelpers/db/piccolo.py +++ b/apphelpers/db/piccolo.py @@ -1,17 +1,9 @@ from contextlib import asynccontextmanager from functools import wraps -from typing import ( - List, - Set, - Type, -) +from typing import List, Set, Type from piccolo.engine.postgres import PostgresEngine -from piccolo.table import ( - Table, - create_db_tables_sync, - drop_db_tables_sync, -) +from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync @asynccontextmanager diff --git a/apphelpers/rest/common.py b/apphelpers/rest/common.py index 44cd5d2..d4a4470 100644 --- a/apphelpers/rest/common.py +++ b/apphelpers/rest/common.py @@ -1,15 +1,7 @@ from __future__ import annotations -from dataclasses import ( - asdict, - dataclass, - field, -) -from typing import ( - Dict, - List, - Optional, -) +from dataclasses import asdict, dataclass, field +from typing import Dict, List, Optional from converge import settings from requests.exceptions import HTTPError diff --git a/apphelpers/rest/fastapi.py b/apphelpers/rest/fastapi.py index 23b8096..c592eab 100644 --- a/apphelpers/rest/fastapi.py +++ b/apphelpers/rest/fastapi.py @@ -3,11 +3,7 @@ from typing import Annotated from converge import settings -from fastapi import ( - APIRouter, - Depends, - Header, -) +from fastapi import APIRouter, Depends, Header from fastapi.routing import APIRoute from starlette.requests import Request @@ -20,11 +16,7 @@ InvalidSessionError, ) from apphelpers.rest import endpoint as ep -from apphelpers.rest.common import ( - User, - notify_honeybadger, - phony, -) +from apphelpers.rest.common import User, notify_honeybadger, phony from apphelpers.sessions import SessionDBHandler if settings.get("HONEYBADGER_API_KEY"): diff --git a/apphelpers/rest/hug.py b/apphelpers/rest/hug.py index acd5233..818c03c 100644 --- a/apphelpers/rest/hug.py +++ b/apphelpers/rest/hug.py @@ -4,11 +4,7 @@ import hug from converge import settings -from falcon import ( - HTTPForbidden, - HTTPNotFound, - HTTPUnauthorized, -) +from falcon import HTTPForbidden, HTTPNotFound, HTTPUnauthorized from hug.decorators import wraps from apphelpers.db import dbtransaction diff --git a/apphelpers/utilities/caching.py b/apphelpers/utilities/caching.py index e82a71e..cb4229e 100644 --- a/apphelpers/utilities/caching.py +++ b/apphelpers/utilities/caching.py @@ -1,12 +1,7 @@ from __future__ import annotations import json -from typing import ( - Any, - ClassVar, - List, - Optional, -) +from typing import Any, ClassVar, List, Optional from redis import Redis diff --git a/docs/conf.py b/docs/conf.py index 299289e..da0249d 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ sys.path.insert(0, os.path.abspath("..")) -import apphelpers +import apphelpers # noqa E402 # -- General configuration --------------------------------------------- diff --git a/fastapi_tests/app/endpoints.py b/fastapi_tests/app/endpoints.py index 60a515e..70ef1c4 100644 --- a/fastapi_tests/app/endpoints.py +++ b/fastapi_tests/app/endpoints.py @@ -4,13 +4,7 @@ from pydantic import BaseModel from apphelpers.rest import endpoint as ep -from apphelpers.rest.fastapi import ( - header, - json_body, - user, - user_agent, - user_id, -) +from apphelpers.rest.fastapi import header, json_body, user, user_agent, user_id from fastapi_tests.app.models import Book diff --git a/fastapi_tests/service.py b/fastapi_tests/service.py index 9bb8c51..06588f7 100644 --- a/fastapi_tests/service.py +++ b/fastapi_tests/service.py @@ -5,10 +5,10 @@ sys.path.append(os.getcwd() + "/..") -import settings -from apphelpers.rest.fastapi import APIFactory -from fastapi_tests.app.endpoints import setup_routes -from fastapi_tests.app.models import db +import settings # noqa E402 +from apphelpers.rest.fastapi import APIFactory # noqa E402 +from fastapi_tests.app.endpoints import setup_routes # noqa E402 +from fastapi_tests.app.models import db # noqa E402 def make_app(): diff --git a/requirements_dev.txt b/requirements_dev.txt index a6af1d4..fd5a2d1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -22,6 +22,7 @@ pre-commit fastapi httpx pytest +pytz uvicorn celery requests diff --git a/setup.cfg b/setup.cfg index 2e761c1..c209e22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.96.1 +current_version = 0.96.2 commit = True tag = True @@ -16,5 +16,10 @@ universal = 1 [flake8] exclude = docs +max-line-length = 88 + +[isort] +multi_line_output = 3 +profile = black [aliases] diff --git a/setup.py b/setup.py index 7ce309e..00e5bf9 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,6 @@ test_suite="tests", tests_require=test_requirements, url="https://github.com/scrolltech/apphelpers", - version="0.96.1", + version="0.96.2", zip_safe=False, ) diff --git a/tests/test_peewee.py b/tests/test_peewee.py index f8a7544..4088e3d 100644 --- a/tests/test_peewee.py +++ b/tests/test_peewee.py @@ -1,11 +1,7 @@ from peewee import TextField import settings -from apphelpers.db.peewee import ( - create_base_model, - create_pgdb_pool, - dbtransaction, -) +from apphelpers.db.peewee import create_base_model, create_pgdb_pool, dbtransaction db = create_pgdb_pool( host=settings.DB_HOST, diff --git a/tests/test_rest.py b/tests/test_rest.py index 30ebb81..d3dbdff 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -1,9 +1,8 @@ -import os -import time from unittest import mock +import falcon +import hug import pytest -import requests from converge import settings from requests.exceptions import HTTPError @@ -11,17 +10,17 @@ from apphelpers.errors.hug import BaseError from apphelpers.rest.hug import honeybadger_wrapper +from . import service from .app.models import globalgroups, sitegroups class urls: - base = "http://127.0.0.1:8000/" - echo = base + "echo" - echo_for_registered = base + "secure-echo" - echo_for_groups = base + "echo-groups" - echo_for_sitegroups = base + "sites/1/echo-groups" - echo_for_all_groups = base + "sites/1/echo-all-groups" - echo_for_custom_authorization = base + "custom-authorization-echo" + echo = "echo" + echo_for_registered = "secure-echo" + echo_for_groups = "echo-groups" + echo_for_sitegroups = "sites/1/echo-groups" + echo_for_all_groups = "sites/1/echo-all-groups" + echo_for_custom_authorization = "custom-authorization-echo" pid_path = "tests/run/app.pid" @@ -36,55 +35,38 @@ class urls: sessionsdb.destroy_all() -def gunicorn_setup_module(): # not working - if os.path.exists(pid_path): - os.remove(pid_path) - cmd = f"gunicorn tests.service:__hug_wsgi__ -p {pid_path} -D" - os.system(cmd) - for i in range(10): - if os.path.exists(pid_path): - time.sleep(2) - break - - -def gunicorn_teardown_module(): - if os.path.exists(pid_path): - cmd = f"kill -9 `cat {pid_path}`" - os.system(cmd) - - def test_get(): word = "hello" url = urls.echo + "/" + word - assert requests.get(url).json() == word + assert hug.test.get(service, url).data == word def test_get_params(): word = "hello" url = urls.echo + "/" + word params = {"word": word} - assert requests.get(url, params=params).json() == word + assert hug.test.get(service, url, params=params).data == word def test_get_multi_params(): nums = [3, 5] - url = urls.base + "add" + url = "add" params = {"nums": nums} - assert requests.get(url, params=params).json() == sum(nums) + assert hug.test.get(service, url, params=params).data == sum(nums) def test_post(): word = "hello" url = urls.echo - assert requests.post(url, json={"word": word}).json() == word + assert hug.test.post(service, url, body={"word": word}).data == word def test_echo_for_registered(): word = "hello" headers = {"NoAuthorization": "Header"} url = urls.echo_for_registered + "/" + word - resp = requests.get(url, headers=headers) - assert resp.status_code == 401 + resp = hug.test.get(service, url, headers=headers) + assert resp.status == falcon.HTTP_UNAUTHORIZED def test_user_id(): @@ -96,21 +78,21 @@ def test_user_id(): word = "hello" url = urls.echo + "/" + word - assert requests.get(url, headers=headers).json() == ("%s:%s" % (uid, word)) + assert hug.test.get(service, url, headers=headers).data == ("%s:%s" % (uid, word)) - url = urls.base + "me/uid" + url = "me/uid" data = {"uid": None} - resp = requests.post(url, json=data, headers=headers) - assert resp.json() == uid + resp = hug.test.post(service, url, body=data, headers=headers) + assert resp.data == uid data = {"uid": 1} # invalid claim - resp = requests.post(url, json=data, headers=headers) - assert resp.json() == uid + resp = hug.test.post(service, url, body=data, headers=headers) + assert resp.data == uid data = {"uid": uid} - resp = requests.post(url, json=data, headers=headers) - assert resp.json() == uid + resp = hug.test.post(service, url, body=data, headers=headers) + assert resp.data == uid def test_group_access(): @@ -122,7 +104,7 @@ def test_group_access(): url = urls.echo_for_groups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 2. Forbidden group uid = 112 @@ -132,7 +114,7 @@ def test_group_access(): url = urls.echo_for_groups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 3. Access group uid = 113 @@ -141,7 +123,7 @@ def test_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 200 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK # 4. Other groups uid = 112 @@ -151,7 +133,7 @@ def test_group_access(): url = urls.echo_for_groups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN def test_not_found(): @@ -161,19 +143,19 @@ def test_not_found(): headers = {"Authorization": sid} - url = urls.base + "snakes/" - assert requests.get(url).status_code == 404 + url = "snakes/" + assert hug.test.get(service, url).status == falcon.HTTP_NOT_FOUND - url = urls.base + "snakes/viper" - resp = requests.get(url) - assert resp.status_code == 200 - assert resp.json() == "viper" + url = "snakes/viper" + resp = hug.test.get(service, url) + assert resp.status == falcon.HTTP_OK + assert resp.data == "viper" - url = urls.base + "sites/1/snakes/" - assert requests.get(url, headers=headers).status_code == 404 + url = "sites/1/snakes/" + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_NOT_FOUND - url = urls.base + "sites/1/snakes/viper" - assert requests.get(url, headers=headers).status_code == 200 + url = "sites/1/snakes/viper" + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK def test_site_group_access(): @@ -186,7 +168,7 @@ def test_site_group_access(): url = urls.echo_for_sitegroups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 2. Forbidden group uid = 115 @@ -197,7 +179,7 @@ def test_site_group_access(): url = urls.echo_for_sitegroups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 3. Access group uid = 116 @@ -207,7 +189,7 @@ def test_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 200 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK def test_all_site_group_access(): @@ -221,7 +203,7 @@ def test_all_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 2. Forbidden group uid = 115 @@ -231,7 +213,7 @@ def test_all_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 3. Access group uid = 116 @@ -241,7 +223,7 @@ def test_all_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 200 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK def test_bound_site_group_access(): @@ -254,7 +236,7 @@ def test_bound_site_group_access(): url = urls.echo_for_sitegroups headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN # 2. Access group uid = 122 @@ -264,7 +246,7 @@ def test_bound_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 200 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK # 2. Access group of Unbound site uid = 123 @@ -274,7 +256,9 @@ def test_bound_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 401 + assert ( + hug.test.get(service, url, headers=headers).status == falcon.HTTP_UNAUTHORIZED + ) uid = 123 groups = [globalgroups.privileged.value] @@ -283,21 +267,25 @@ def test_bound_site_group_access(): sid = sessionsdb.create(**d) headers = {"Authorization": sid} - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN def test_request_access(): - url = urls.base + "request-and-body" - req = requests.post(url, data={"z": 1}, headers={"testheader": "testheader-value"}) - resp = req.json() + url = "request-and-body" + req = hug.test.post( + service, url, body={"z": 1}, headers={"testheader": "testheader-value"} + ) + resp = req.data assert "testheader".upper() in resp["headers"] - assert resp["body"] == {"z": "1"} + assert resp["body"] == {"z": 1} def test_raw_request(): - url = urls.base + "request-raw-body" - req = requests.post(url, data={"z": 1}, headers={"testheader": "testheader-value"}) - resp = req.json() + url = "request-raw-body" + req = hug.test.post( + service, url, body={"z": 1}, headers={"testheader": "testheader-value"} + ) + resp = req.data assert "testheader".upper() in resp["headers"] @@ -309,13 +297,13 @@ def test_custom_authorization_access(): headers = {"Authorization": sid} url = urls.echo_for_custom_authorization + "/authorized" - assert requests.get(url).status_code == 401 + assert hug.test.get(service, url).status == falcon.HTTP_UNAUTHORIZED url = urls.echo_for_custom_authorization + "/authorized" - assert requests.get(url, headers=headers).status_code == 200 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_OK url = urls.echo_for_custom_authorization + "/unauthorized" - assert requests.get(url, headers=headers).status_code == 403 + assert hug.test.get(service, url, headers=headers).status == falcon.HTTP_FORBIDDEN def test_honeybadger_wrapper():