move pymssql back to an optional requirement (#4581)

Co-authored-by: Neville Samuell <[email protected]>

CHANGELOG.md

@@ -26,6 +26,7 @@ The types of changes are:
 
 - Delay rendering the nav until all necessary queries are finished loading [#4571](https://github.com/ethyca/fides/pull/4571)
 - Updating return value for crud.get_custom_fields_filtered [#4575](https://github.com/ethyca/fides/pull/4575)
+- Moved `pymssl` to an optional dependency no longer installed by default with our python package [#4581](https://github.com/ethyca/fides/pull/4581)
 
 ### Fixed
 

Dockerfile

@@ -33,6 +33,7 @@ RUN apt-get update && \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Python Dependencies
+
 COPY dev-requirements.txt .
 RUN pip install --user -U pip --no-cache-dir install -r dev-requirements.txt
 
@@ -44,8 +45,9 @@ ENV PATH="/opt/fides/bin:${PATH}"
 RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
 
 COPY requirements.txt .
-
 RUN pip install --no-cache-dir install -r requirements.txt
+COPY optional-requirements.txt .
+RUN pip install --no-cache-dir install -r optional-requirements.txt
 
 COPY dev-requirements.txt .
 RUN pip install --no-cache-dir install -r dev-requirements.txt

MANIFEST.in

@@ -2,6 +2,7 @@ include LICENSE
 include README.md
 include requirements.txt
 include dev-requirements.txt
+include optional-requirements.txt
 include versioneer.py
 include src/fides/api/alembic/alembic.ini
 include src/fides/_version.py

README.md

@@ -27,21 +27,6 @@ In order to get started quickly with Fides, a sample project is bundled within t
 * [Docker](https://www.docker.com/products/docker-desktop) (version 20.10.11 or later)
 * [Python](https://www.python.org/downloads/) (version 3.8 through 3.10)
 
-#### Additional requirements (for ARM Mac Users)
-
-Due to platform differences, the following dependencies and steps are also required:
-
-```bash
-brew install freetds openssl
-```
-
-**Add the following to your run commands (i.e. `.zshrc`), updating any path/versions to match yours**
-
-```bash
-export LDFLAGS="-L/opt/homebrew/Cellar/freetds/1.3.18/lib -L/opt/homebrew/Cellar/[email protected]/1.1.1u/lib"`
-export CFLAGS="-I/opt/homebrew/Cellar/freetds/1.3.18/include"
-```
-
 #### Download and install Fides
 
 You can easily download and install Fides using `pip`. Run the following command to get started:
@@ -131,3 +116,24 @@ Fides is created and sponsored by Ethyca: a developer tools company building the
 [mypy-url]: http://mypy-lang.org/
 [twitter-image]: https://img.shields.io/twitter/follow/ethyca?style=social
 [twitter-url]: https://twitter.com/ethyca
+
+## ⚠️ Advanced Setup for Microsoft SQL Server (MSSQL) Support
+
+By default, running `pip install ethyca-fides` locally will not install the optional Python libraries needed for Microsoft SQL Server, since these rely on additional system dependencies (`freetds`)! However, if you *do* want to connect to MSSQL, you have two options:
+1. Use our pre-built Docker images which install these optional dependencies automatically: [`ethyca/fides`](https://hub.docker.com/r/ethyca/fides). See our [Deployment Guide](https://ethyca.com/docs/dev-docs/configuration/deployment) for more!
+2. Install the required dependencies on your local development machine and run `pip install ethyca-fides[all]` to include "all" the optional libraries. Keep reading to learn more about this!
+
+For local development setup on macOS, follow these steps:
+1. Install the required development libraries from Homebrew:
+```bash
+brew install freetds openssl
+```
+2. Add the following to your shell (i.e. `.zshrc`) to ensure your compiler can access the `freetds` and `openssl` libraries, updating the paths & versions to match your local install:
+```bash
+export LDFLAGS="-L/opt/homebrew/Cellar/freetds/1.3.18/lib -L/opt/homebrew/Cellar/[email protected]/1.1.1u/lib"`
+export CFLAGS="-I/opt/homebrew/Cellar/freetds/1.3.18/include"
+```
+3. Reinstall Fides with MSSQL support by including the `all` extra requirement:
+```bash
+pip install ethyca-fides[all]
+```

noxfiles/ci_nox.py

@@ -2,9 +2,9 @@
 from functools import partial
 from typing import Callable, Dict
 
-import nox
 from nox.command import CommandFailed
 
+import nox
 from constants_nox import (
     CONTAINER_NAME,
     IMAGE_NAME,
@@ -312,7 +312,7 @@ def collect_tests(session: nox.Session) -> None:
     errors within the test code.
     """
     session.install(".")
-    install_requirements(session)
+    install_requirements(session, True)
     command = ("pytest", "tests/", "--collect-only")
     session.run(*command)
 

noxfiles/utils_nox.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 import nox
-
 from constants_nox import COMPOSE_FILE_LIST
 from run_infrastructure import run_infrastructure
 
@@ -51,9 +50,11 @@ def teardown(session: nox.Session, volumes: bool = False, images: bool = False)
     session.log("Teardown complete")
 
 
-def install_requirements(session: nox.Session) -> None:
+def install_requirements(session: nox.Session, include_optional: bool = False) -> None:
     session.install("-r", "requirements.txt")
     session.install("-r", "dev-requirements.txt")
+    if include_optional:
+        session.install("-r", "optional-requirements.txt")
 
 
 @nox.session()

optional-requirements.txt

@@ -0,0 +1 @@
+pymssql==2.2.8

requirements.txt

@@ -40,7 +40,6 @@ pydash==6.0.2
 PyJWT==2.4.0
 pymongo==3.13.0
 PyMySQL==1.0.2
-pymssql==2.2.8
 python-jose[cryptography]==3.3.0
 pyyaml==6.0.1
 redis==3.5.3

setup.py

@@ -1,4 +1,5 @@
 import pathlib
+from typing import List
 
 from setuptools import find_packages, setup
 
@@ -13,6 +14,41 @@
 
 install_requires = open("requirements.txt", encoding="utf-8").read().strip().split("\n")
 dev_requires = open("dev-requirements.txt", encoding="utf-8").read().strip().split("\n")
+optional_requires = (
+    open("optional-requirements.txt", encoding="utf-8").read().strip().split("\n")
+)
+
+
+def optional_requirements(
+    dependency_names: List[str], requires: List[str] = optional_requires
+) -> List[str]:
+    """
+    Matches the provided dependency names to lines in `optional-requirements.txt`,
+    and returns the full dependency string for each one.
+    Prevents the need to store version numbers in two places.
+    """
+
+    requirements: List[str] = []
+
+    for dependency in dependency_names:
+        for optional_dependency in requires:
+            if optional_dependency.startswith(dependency):
+                requirements.append(optional_dependency)
+                break
+
+    if len(requirements) == len(dependency_names):
+        return requirements
+
+    raise ModuleNotFoundError
+
+
+# Human-Readable Extras
+# Versions are read from corresponding lines in `optional-requirements.txt`
+extras = {
+    "mssql": optional_requirements(["pymssql"], optional_requires),
+}
+
+extras["all"] = sum([value for value in extras.values()], [])
 
 ###################
 ## Package Setup ##