From 8c8d6f0f1775357811001b88c8f07442e5f32b19 Mon Sep 17 00:00:00 2001 From: John Rofrano Date: Sat, 9 Jul 2022 21:57:24 +0000 Subject: [PATCH] Fixed linting warnings and added pylint --- dot-env-example => .flaskenv | 2 +- .gitattributes | 3 ++ Makefile | 58 ++++++++++++++++++++++----- service/__init__.py | 16 ++++---- service/models.py | 20 ++++----- service/routes.py | 28 +++++++------ service/{ => utils}/error_handlers.py | 48 +++++++++++++++++++++- service/utils/log_handlers.py | 36 +++++++++++++++++ service/{ => utils}/status.py | 13 ++++++ tests/test_models.py | 15 ++++--- tests/test_routes.py | 34 ++++++++-------- 11 files changed, 204 insertions(+), 69 deletions(-) rename dot-env-example => .flaskenv (51%) create mode 100644 .gitattributes rename service/{ => utils}/error_handlers.py (52%) create mode 100644 service/utils/log_handlers.py rename service/{ => utils}/status.py (74%) diff --git a/dot-env-example b/.flaskenv similarity index 51% rename from dot-env-example rename to .flaskenv index a7784ad..726be1b 100644 --- a/dot-env-example +++ b/.flaskenv @@ -1,2 +1,2 @@ -PORT=8080 +FLASK_RUN_PORT=8080 FLASK_APP=service:app \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5dc46e6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf \ No newline at end of file diff --git a/Makefile b/Makefile index 04805e4..9b4f5c2 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,74 @@ -.PHONY: all help install venv test run +# These can be overidden with env vars. +REGISTRY ?= rofrano +IMAGE_NAME ?= hitcounter +IMAGE_TAG ?= 1.0 +IMAGE ?= $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) +PLATFORM ?= "linux/amd64,linux/arm64" +.PHONY: help help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-\\.]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +.PHONY: all all: help +.PHONY: clean +clean: ## Removes all dangling build cache + $(info Removing all dangling build cache..) + -docker rmi $(IMAGE) + docker image prune -f + docker buildx prune -f + +.PHONY: venv venv: ## Create a Python virtual environment $(info Creating Python 3 virtual environment...) python3 -m venv .venv +.PHONY: install install: ## Install dependencies $(info Installing dependencies...) sudo pip install -r requirements.txt +.PHONY: lint lint: ## Run the linter $(info Running linting...) - flake8 service --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 service --count --max-complexity=10 --max-line-length=127 --statistics + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --max-complexity=10 --max-line-length=127 --statistics + pylint service +.PHONY: test test: ## Run the unit tests $(info Running tests...) nosetests --with-spec --spec-color +.PHONY: run run: ## Run the service $(info Starting service...) honcho start -depoy: ## Deploy the service on Kubernetes - $(info Deploying service...) - kubectl apply -f kube/redis.yaml - kubectl apply -f kube/secrets.yaml - kubectl apply -f kube/deployment.yaml - kubectl apply -f kube/service.yaml +.PHONY: deploy +depoy: ## Deploy the service on local Kubernetes + $(info Deploying service locally...) + kubectl apply -k kube/overlay/local + +############################################################ +# COMMANDS FOR BUILDING THE IMAGE +############################################################ + +.PHONY: init +init: export DOCKER_BUILDKIT=1 +init: ## Creates the buildx instance + $(info Initializing Builder...) + docker buildx create --use --name=qemu + docker buildx inspect --bootstrap + +.PHONY: build +build: ## Build all of the project Docker images + $(info Building $(IMAGE) for $(PLATFORM)...) + docker buildx build --file Dockerfile --pull --platform=$(PLATFORM) --tag $(IMAGE) --push . + +.PHONY: remove +remove: ## Stop and remove the buildx builder + $(info Stopping and removing the builder image...) + docker buildx stop + docker buildx rm diff --git a/service/__init__.py b/service/__init__.py index 65cdee8..af0f775 100644 --- a/service/__init__.py +++ b/service/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2016, 2020 John J. Rofrano. All Rights Reserved. +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import os import logging from flask import Flask +from service.utils import log_handlers # NOTE: Do not change the order of this code # The Flask app must be created @@ -28,16 +29,13 @@ # Create the Flask aoo app = Flask(__name__) -# Import the routes After the Flask app is created -from service import routes, models, error_handlers +# Dependencies require we import the routes AFTER the Flask app is created +# pylint: disable=wrong-import-position, wrong-import-order, cyclic-import +from service import routes +from service.utils import error_handlers # noqa: F401, E402 # Set up logging for production -app.logger.propagate = False -if __name__ != "__main__": - gunicorn_logger = logging.getLogger("gunicorn.error") - if gunicorn_logger: - app.logger.handlers = gunicorn_logger.handlers - app.logger.setLevel(gunicorn_logger.level) +log_handlers.init_logging(app, "gunicorn.error") app.logger.info(70 * "*") app.logger.info(" H I T C O U N T E R S E R V I C E ".center(70, "*")) diff --git a/service/models.py b/service/models.py index be6e600..382b38d 100644 --- a/service/models.py +++ b/service/models.py @@ -1,5 +1,5 @@ ###################################################################### -# Copyright 2016, 2020 John Rofrano. All Rights Reserved. +# Copyright 2016, 2022 John Rofrano. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. @@ -19,18 +19,18 @@ import os import logging from redis import Redis -from redis.exceptions import ConnectionError +from redis.exceptions import ConnectionError as RedisConnectionError logger = logging.getLogger(__name__) DATABASE_URI = os.getenv("DATABASE_URI", "redis://localhost:6379") -class DatabaseConnectionError(ConnectionError): - pass +class DatabaseConnectionError(RedisConnectionError): + """Indicates that a database connection error has occurred""" -class Counter(object): +class Counter(): """An integer counter that is persisted in Redis You can establish a connection to Redis using an environment @@ -71,6 +71,7 @@ def increment(self): return Counter.redis.incr(self.name) def serialize(self): + """Creates a Python dictionary from the instance""" return dict(name=self.name, counter=int(Counter.redis.get(self.name))) ###################################################################### @@ -86,7 +87,7 @@ def all(cls): for key in cls.redis.keys("*") ] except Exception as err: - raise DatabaseConnectionError(err) + raise DatabaseConnectionError(err) from err return counters @classmethod @@ -98,15 +99,16 @@ def find(cls, name): if count: counter = Counter(name, count) except Exception as err: - raise DatabaseConnectionError(err) + raise DatabaseConnectionError(err) from err return counter @classmethod def remove_all(cls): + """Deletes all of the counters""" try: cls.redis.flushall() except Exception as err: - raise DatabaseConnectionError(err) + raise DatabaseConnectionError(err) from err ###################################################################### # R E D I S D A T A B A S E C O N N E C T I O N M E T H O D S @@ -120,7 +122,7 @@ def test_connection(cls): cls.redis.ping() logger.info("Connection established") success = True - except ConnectionError: + except RedisConnectionError: logger.warning("Connection Error!") return success diff --git a/service/routes.py b/service/routes.py index 96db59b..c1a3091 100644 --- a/service/routes.py +++ b/service/routes.py @@ -1,4 +1,4 @@ -# Copyright 2015, 2021 John J. Rofrano All Rights Reserved. +# Copyright 2015, 2022 John J. Rofrano All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,10 @@ """ Redis Counter Demo in Docker """ -import os from flask import jsonify, abort, url_for -from . import app, status # HTTP Status Codes -from service import DATABASE_URI -from .models import Counter, DatabaseConnectionError - -DEBUG = os.getenv("DEBUG", "False") == "True" -PORT = os.getenv("PORT", "8080") +from service import app, DATABASE_URI +from service.utils import status # HTTP Status Codes +from service.models import Counter, DatabaseConnectionError ############################################################ @@ -47,6 +43,7 @@ def index(): ############################################################ @app.route("/counters", methods=["GET"]) def list_counters(): + """Lists all counters in the database""" app.logger.info("Request to list all counters...") try: counters = Counter.all() @@ -61,7 +58,8 @@ def list_counters(): ############################################################ @app.route("/counters/", methods=["GET"]) def read_counters(name): - app.logger.info("Request to Read counter: {}...".format(name)) + """Reads a counter from the database""" + app.logger.info("Request to Read counter: %s...", name) try: counter = Counter.find(name) @@ -69,9 +67,9 @@ def read_counters(name): abort(status.HTTP_503_SERVICE_UNAVAILABLE, err) if not counter: - abort(status.HTTP_404_NOT_FOUND, "Counter {} does not exist".format(name)) + abort(status.HTTP_404_NOT_FOUND, f"Counter [{name}] does not exist") - app.logger.info("Returning: {}...".format(counter.value)) + app.logger.info("Returning: %d...", counter.value) return jsonify(counter.serialize()) @@ -80,11 +78,12 @@ def read_counters(name): ############################################################ @app.route("/counters/", methods=["POST"]) def create_counters(name): + """Creates a counter in the database""" app.logger.info("Request to Create counter...") try: counter = Counter.find(name) if counter is not None: - return jsonify(code=409, error="Counter already exists"), 409 + return jsonify(code=409, error=f"Counter [{name}] already exists"), 409 counter = Counter(name) except DatabaseConnectionError as err: @@ -103,12 +102,13 @@ def create_counters(name): ############################################################ @app.route("/counters/", methods=["PUT"]) def update_counters(name): + """Updates a counter in the database""" app.logger.info("Request to Update counter...") try: counter = Counter.find(name) if counter is None: return ( - jsonify(code=404, error="Counter {} does not exist".format(name)), + jsonify(code=404, error=f"Counter [{name}] does not exist"), 404, ) @@ -124,6 +124,7 @@ def update_counters(name): ############################################################ @app.route("/counters/", methods=["DELETE"]) def delete_counters(name): + """Removes te counter from teh database""" app.logger.info("Request to Delete counter...") try: counter = Counter.find(name) @@ -142,6 +143,7 @@ def delete_counters(name): @app.before_first_request def init_db(): + """Initialize the Redis database""" try: app.logger.info("Initializing the Redis database") Counter.connect(DATABASE_URI) diff --git a/service/error_handlers.py b/service/utils/error_handlers.py similarity index 52% rename from service/error_handlers.py rename to service/utils/error_handlers.py index 8f05821..423c36c 100644 --- a/service/error_handlers.py +++ b/service/utils/error_handlers.py @@ -1,9 +1,38 @@ +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module: error_handlers +""" from flask import jsonify -from . import app, status +from service import app +from . import status + ###################################################################### # Error Handlers ###################################################################### +# @app.errorhandler(status.HTTP_400_BAD_REQUEST) +# def bad_request(error): +# """Handles bad requests with 400_BAD_REQUEST""" +# message = str(error) +# app.logger.warning(message) +# return ( +# jsonify( +# status=status.HTTP_400_BAD_REQUEST, error="Bad Request", message=message +# ), +# status.HTTP_400_BAD_REQUEST, +# ) @app.errorhandler(status.HTTP_404_NOT_FOUND) @@ -32,6 +61,21 @@ def method_not_supported(error): ) +# @app.errorhandler(status.HTTP_415_UNSUPPORTED_MEDIA_TYPE) +# def mediatype_not_supported(error): +# """Handles unsupported media requests with 415_UNSUPPORTED_MEDIA_TYPE""" +# message = str(error) +# app.logger.warning(message) +# return ( +# jsonify( +# status=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, +# error="Unsupported media type", +# message=message, +# ), +# status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, +# ) + + @app.errorhandler(status.HTTP_500_INTERNAL_SERVER_ERROR) def internal_server_error(error): """Handles unexpected server error with 500_SERVER_ERROR""" @@ -49,7 +93,7 @@ def internal_server_error(error): @app.errorhandler(status.HTTP_503_SERVICE_UNAVAILABLE) def service_unavailable(error): - """Handles unexpected server error with 503_SERVICE_UNAVAILABLE""" + """ Handles unexpected server error with 503_SERVICE_UNAVAILABLE """ message = str(error) app.logger.error(message) return ( diff --git a/service/utils/log_handlers.py b/service/utils/log_handlers.py new file mode 100644 index 0000000..3034347 --- /dev/null +++ b/service/utils/log_handlers.py @@ -0,0 +1,36 @@ +###################################################################### +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### + +""" +Log Handlers + +This module contains utility functions to set up logging +consistently +""" +import logging + + +def init_logging(app, logger_name: str): + """Set up logging for production""" + app.logger.propagate = False + gunicorn_logger = logging.getLogger(logger_name) + app.logger.handlers = gunicorn_logger.handlers + app.logger.setLevel(gunicorn_logger.level) + # Make all log formats consistent + formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] [%(module)s] %(message)s", "%Y-%m-%d %H:%M:%S %z") + for handler in app.logger.handlers: + handler.setFormatter(formatter) + app.logger.info("Logging handler established") diff --git a/service/status.py b/service/utils/status.py similarity index 74% rename from service/status.py rename to service/utils/status.py index d3d655e..12f13c9 100644 --- a/service/status.py +++ b/service/utils/status.py @@ -1,4 +1,17 @@ # coding: utf8 +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Descriptive HTTP status codes, for code readability. See RFC 2616 and RFC 6585. diff --git a/tests/test_models.py b/tests/test_models.py index 80fc34a..d70c51d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016, 2021 John J. Rofrano. All Rights Reserved. +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import logging from unittest import TestCase from unittest.mock import patch -from redis.exceptions import ConnectionError +from redis.exceptions import ConnectionError as RedisConnectionError from service.models import Counter, DatabaseConnectionError DATABASE_URI = os.getenv("DATABASE_URI", "redis://:@localhost:6379/0") @@ -52,7 +52,6 @@ def setUp(self): def tearDown(self): """This runs after each test""" # Counter.redis.flushall() - pass ###################################################################### # T E S T C A S E S @@ -89,13 +88,13 @@ def test_set_find_counter(self): """Find a counter""" _ = Counter("foo") _ = Counter("bar") - foo = Counter.find("foo") - self.assertEqual(foo.name, "foo") + found = Counter.find("foo") + self.assertEqual(found.name, "foo") def test_counter_not_found(self): """counter not found""" - foo = Counter.find("foo") - self.assertIsNone(foo) + found = Counter.find("foo") + self.assertIsNone(found) def test_set_get_counter(self): """Set and then Get the counter""" @@ -137,7 +136,7 @@ def test_increment_counter_to_2(self): @patch("redis.Redis.ping") def test_no_connection(self, ping_mock): """Handle failed connection""" - ping_mock.side_effect = ConnectionError() + ping_mock.side_effect = RedisConnectionError() self.assertRaises(DatabaseConnectionError, self.counter.connect, DATABASE_URI) @patch.dict(os.environ, {"DATABASE_URI": ""}) diff --git a/tests/test_routes.py b/tests/test_routes.py index 7487ee7..4aba587 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016, 2020 John J. Rofrano. All Rights Reserved. +# Copyright 2016, 2022 John J. Rofrano. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,7 +47,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """This runs once after the entire test suite""" - pass def setUp(self): """This runs before each test""" @@ -57,7 +56,6 @@ def setUp(self): def tearDown(self): """This runs after each test""" - pass ###################################################################### # T E S T C A S E S @@ -66,37 +64,37 @@ def tearDown(self): def test_index(self): """Get the home page""" resp = self.app.get("/") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) def test_health(self): """Get the health endpoint""" resp = self.app.get("/health") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) data = resp.get_json() self.assertEqual(data["status"], "OK") def test_create_counter(self): """Create a counter""" resp = self.app.post("/counters/foo") - self.assertEquals(resp.status_code, 201) + self.assertEqual(resp.status_code, 201) data = resp.get_json() self.assertEqual(data["counter"], 0) def test_counter_already_exists(self): """Counter already exists""" resp = self.app.post("/counters/foo") - self.assertEquals(resp.status_code, 201) + self.assertEqual(resp.status_code, 201) resp = self.app.post("/counters/foo") - self.assertEquals(resp.status_code, 409) + self.assertEqual(resp.status_code, 409) def test_list_counters(self): """Get the counter""" resp = self.app.post("/counters/foo") - self.assertEquals(resp.status_code, 201) + self.assertEqual(resp.status_code, 201) resp = self.app.post("/counters/bar") - self.assertEquals(resp.status_code, 201) + self.assertEqual(resp.status_code, 201) resp = self.app.get("/counters") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) data = resp.get_json() self.assertEqual(len(data), 2) @@ -104,30 +102,30 @@ def test_get_counter(self): """Get the counter""" self.test_create_counter() resp = self.app.get("/counters/foo") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) data = resp.get_json() self.assertEqual(data["counter"], 0) def test_get_counter_not_found(self): """Test counter not found""" resp = self.app.get("/counters/foo") - self.assertEquals(resp.status_code, 404) + self.assertEqual(resp.status_code, 404) def test_put_counter_not_found(self): """Test counter not found""" resp = self.app.put("/counters/foo") - self.assertEquals(resp.status_code, 404) + self.assertEqual(resp.status_code, 404) def test_increment_counter(self): """Increment the counter""" self.test_get_counter() resp = self.app.put("/counters/foo") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) data = resp.get_json() self.assertEqual(data["counter"], 1) resp = self.app.put("/counters/foo") - self.assertEquals(resp.status_code, 200) + self.assertEqual(resp.status_code, 200) data = resp.get_json() logging.debug(data) self.assertEqual(data["counter"], 2) @@ -136,12 +134,12 @@ def test_delete_counter(self): """Delete the counter""" self.test_create_counter() resp = self.app.delete("/counters/foo") - self.assertEquals(resp.status_code, 204) + self.assertEqual(resp.status_code, 204) def test_method_not_allowed(self): """Test Method Not Allowed""" resp = self.app.post("/counters") - self.assertEquals(resp.status_code, 405) + self.assertEqual(resp.status_code, 405) ###################################################################### # T E S T E R R O R H A N D L E R S