Introduce Linting

.github/workflows/build-deploy.yaml

@@ -16,6 +16,45 @@ jobs:
       flake: true
       black: true
       pythonVersion: 3.11-buster
+      skipDjangoCheck: true
+
+  linting:
+    name: Linting Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.local/share/virtualenvs
+          key: v0-${{ hashFiles('./Pipfile.lock') }}
+      - name: Install Dependencies (pipenv)
+        shell: bash
+        continue-on-error: true
+        run: |-
+          cd .
+          pip install pipenv
+          pipenv install --dev
+      - name: Lint (flake8)
+        run: |-
+          cd .
+          pipenv run flake8 .
+      - name: Lint (black)
+        run: |-
+          cd .
+          pipenv run black --check .
+    container:
+      image: python:3.11-buster
+    env:
+      DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_PASSWORD: postgres
+        options: "--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5"
 
   publish-backend:
     uses: pennlabs/shared-actions/.github/workflows/[email protected]

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/pennlabs/pre-commit-hooks
+    rev: v1.0-mobile-backend
+    hooks:
+      - id: black
+        args: []
+      - id: isort
+        args: []
+      - id: flake8
+        args: [--config, setup.cfg]
+      - id: detect-private-key

Pipfile

@@ -19,11 +19,16 @@ passlib = "*"
 jwcrypto = "*"
 pytest-asyncio = "*"
 pytest = "*"
+pre-commit = "*"
+click = "==8.0.4"
 
 [dev-packages]
-isort = "*"
-flake8 = "*"
-black = "*"
+flake8 = "==6.1.0"
+flake8-isort = "==6.1.0"
+flake8-quotes = "==3.3.2"
+flake8-absolute-import = "*"
+black = "==19.10b0"
+coverage = "*"
 
 [requires]
 python_version = "3.11"

README.md

@@ -24,6 +24,11 @@ git clone [email protected]:pennlabs/labs-analytics.git
 pipenv install
 ```
 
+4. Setting up precommit:
+```bash
+pipenv run pre-commit install
+```
+
 ### Setting up for local development
 
 This guide details the steps to set up `Redis`, `Redis Insight`, `postgres` and `pgweb` instances using Docker, making it easy for development.

scripts/create_table.py

@@ -1,27 +1,9 @@
 import asyncio
-from typing import Any
 
 from settings.config import DATABASE_URL
-from sqlalchemy import (
-    Boolean,
-    Column,
-    CursorResult,
-    DateTime,
-    ForeignKey,
-    Identity,
-    Insert,
-    Integer,
-    LargeBinary,
-    MetaData,
-    Select,
-    String,
-    Table,
-    Update,
-    func,
-)
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Column, DateTime, Identity, Integer, MetaData, String, Table
+from sqlalchemy.ext.asyncio import create_async_engine
+
 
 engine = create_async_engine(str(DATABASE_URL))
 

scripts/flush_db.py

@@ -1,13 +1,12 @@
 import asyncio
 import json
-import sys
 from datetime import datetime
 
 import asyncpg
-from redis.asyncio import Redis
-
 from settings.config import DATABASE_URL, REDIS_BATCH_SIZE, REDIS_URL
 
+from redis.asyncio import Redis
+
 
 async def batch_insert(events):
     BATCH_INSERT_COMMAND = """

scripts/settings/config.py

@@ -1,5 +1,6 @@
 import os
 
+
 REDIS_URL = os.getenv("REDIS_URL")
 REDIS_BATCH_SIZE = 1000
 

setup.cfg

@@ -0,0 +1,17 @@
+[flake8]
+max-line-length = 100
+exclude = .venv, migrations
+inline-quotes = double
+ignore = E203, W503
+
+[isort]
+default_section = THIRDPARTY
+known_first_party = src, scripts
+line_length = 100
+lines_after_imports = 2
+multi_line_output = 3
+include_trailing_comma = True
+use_parentheses = True
+
+[coverage:run]
+source = .

src/auth.py

@@ -4,6 +4,7 @@
 
 from src.config import settings
 
+
 # The URL to the JWKS endpoint
 JWKS_URL = settings.JWKS_URL
 
@@ -34,7 +35,9 @@ def get_token_from_header(request: Request):
             raise HTTPException(status_code=401, detail="Wrong authentication scheme")
         return token
     except ValueError:
-        raise HTTPException(status_code=401, detail="Invalid authorization header format")
+        raise HTTPException(
+            status_code=401, detail="Invalid authorization header format"
+        )
 
 
 def verify_jwt(token: str = Depends(get_token_from_header)):

src/database.py

@@ -1,18 +1,11 @@
 import asyncio
 
-from sqlalchemy import (
-    Column,
-    DateTime,
-    Identity,
-    Integer,
-    MetaData,
-    String,
-    Table,
-)
+from sqlalchemy import Column, DateTime, Identity, Integer, MetaData, String, Table
 from sqlalchemy.ext.asyncio import create_async_engine
 
 from src.config import settings
 
+
 DATABASE_URL = settings.DATABASE_URL
 
 engine = create_async_engine(DATABASE_URL)

src/main.py

@@ -1,8 +1,9 @@
 from fastapi import Depends, FastAPI, HTTPException, Request
 
 from src.auth import verify_jwt
-from src.redis import set_redis_from_tx
 from src.models import AnalyticsTxn
+from src.redis import set_redis_from_tx
+
 
 app = FastAPI()
 

src/models.py

@@ -1,16 +1,15 @@
 import hashlib
-from enum import Enum
+import json
 from datetime import datetime
+from enum import Enum
 from typing import Optional
 
 from fastapi.encoders import jsonable_encoder
-from pydantic import BaseModel, ConfigDict, json
+from pydantic import BaseModel, ConfigDict
 
 
 class CustomModel(BaseModel):
-    model_config = ConfigDict(
-        populate_by_name=True,
-    )
+    model_config = ConfigDict(populate_by_name=True,)
 
     def serializable_dict(self, **kwargs):
         default_dict = self.model_dump()
@@ -48,6 +47,7 @@ class RedisEvent(CustomModel):
     key: bytes | str
     value: bytes | str
 
+
 class AnalyticsTxn(CustomModel):
     product: Product
     pennkey: Optional[str] = None

src/redis.py

@@ -2,13 +2,15 @@
 
 from redis.asyncio import Redis
 from src.config import settings
-from src.models import RedisEvent
-from src.models import AnalyticsTxn
+from src.models import AnalyticsTxn, RedisEvent
+
 
 redis_client: Redis = Redis.from_url(str(settings.REDIS_URL))
 
 
-async def set_redis_keys(data: list[RedisEvent], *, is_transaction: bool = False) -> None:
+async def set_redis_keys(
+    data: list[RedisEvent], *, is_transaction: bool = False
+) -> None:
     async with redis_client.pipeline(transaction=is_transaction) as pipe:
         for redis_data in data:
             await pipe.set(redis_data.key, redis_data.value)

tests/test_load.py

@@ -7,6 +7,7 @@
 import requests
 from test_token import get_tokens
 
+
 # Runtime should be less that 3 seconds for most laptops
 BENCHMARK_TIME = 3  # seconds
 
@@ -54,7 +55,8 @@ def make_request():
 
 def run_threads():
     with ThreadPoolExecutor(max_workers=THREADS) as executor:
-        futures = [executor.submit(make_request) for _ in range(NUMBER_OF_REQUESTS)]
+        for _ in range(NUMBER_OF_REQUESTS):
+            executor.submit(make_request)
 
 
 def test_load():

tests/test_redis.py

@@ -1,7 +1,7 @@
 import pytest
 
-from src.redis import get_by_key, set_redis_keys
 from src.models import RedisEvent
+from src.redis import get_by_key, set_redis_keys
 
 
 @pytest.mark.asyncio

tests/test_token.py

@@ -5,6 +5,7 @@
 
 from src.auth import verify_jwt
 
+
 ATTEST_URL = "https://platform.pennlabs.org/identity/attest/"
 
 # Using Penn Basics DLA Account for testing, will not work if you don't have that in .env