From 69f169aae819b19e7b8e2db4aa0a779ef6629cc5 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Tue, 20 Jul 2021 12:37:43 -0400 Subject: [PATCH 01/19] Add stack trace on exception --- .../handle_notifications/sources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 54141683..5cb16aed 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -6,6 +6,7 @@ import datetime import email import json +import traceback from urllib.parse import urlparse from email.utils import mktime_tz, parsedate_tz from typing import Iterable, Optional, TypeVar, Type, Tuple, Dict, Union @@ -624,8 +625,9 @@ def get_notifications( ) except Exception as error: - job_logger.log_warning( - message=f"Issue fetching notifications from {notification_source.name}: {error}", + stacktrace = traceback.format_exc() + job_logger.log_failure( + message=f"Issue fetching notifications from {notification_source.name}: {error}\n```\n{stacktrace}\n```", ) return received_notifications From cbbe2f3113e97a0c29f1568fb9cbb331f357fe95 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Tue, 20 Jul 2021 13:43:18 -0400 Subject: [PATCH 02/19] Improve notificationsource detail view --- .../nautobot_circuit_maintenance/notificationsource.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nautobot_circuit_maintenance/templates/nautobot_circuit_maintenance/notificationsource.html b/nautobot_circuit_maintenance/templates/nautobot_circuit_maintenance/notificationsource.html index dd021da1..bba92f71 100644 --- a/nautobot_circuit_maintenance/templates/nautobot_circuit_maintenance/notificationsource.html +++ b/nautobot_circuit_maintenance/templates/nautobot_circuit_maintenance/notificationsource.html @@ -44,9 +44,11 @@

{{ object.name }}

Providers - {% for provider in providers %} - {{ provider.name }} - {% endfor %} + {% if authentication_message %} From 6366603f28790da39ffeddf5e7a230364f1899ff Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Tue, 20 Jul 2021 13:44:04 -0400 Subject: [PATCH 03/19] Avoid exception when emails_circuit_maintenances field is not populated --- nautobot_circuit_maintenance/handle_notifications/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 5cb16aed..9872bc02 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -254,7 +254,7 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: ) """ for provider in Provider.objects.all(): - if "emails_circuit_maintenances" in provider.custom_field_data: + if provider.custom_field_data.get("emails_circuit_maintenances"): if email_source in provider.custom_field_data["emails_circuit_maintenances"].split(","): provider_type = provider.slug break From 27e96c57f80809eb8d0c9973f051cef561e83314 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Tue, 20 Jul 2021 17:14:52 -0400 Subject: [PATCH 04/19] Improve some log message formatting --- .../handle_notifications/handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/handler.py b/nautobot_circuit_maintenance/handle_notifications/handler.py index 56ff22e5..0c142ffb 100644 --- a/nautobot_circuit_maintenance/handle_notifications/handler.py +++ b/nautobot_circuit_maintenance/handle_notifications/handler.py @@ -194,16 +194,16 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) break except ParsingError as exc: tb_str = traceback.format_exception(etype=type(exc), value=exc, tb=exc.__traceback__) - logger.log_debug(message=f"Parsing failed for notification {notification.subject}:.\n{tb_str}") + logger.log_debug(message=f"Parsing failed for notification `{notification.subject}`:\n```\n{tb_str}\n```") except Exception as exc: tb_str = traceback.format_exception(etype=type(exc), value=exc, tb=exc.__traceback__) logger.log_debug( - message=f"Unexpected exception while parsing notification {notification.subject}.\n{tb_str}" + message=f"Unexpected exception while parsing notification `{notification.subject}`.\n```\n{tb_str}\n```" ) else: parsed_notifications = [] raw_payload = b"" - logger.log_warning(message=f"Parsed failed for all the raw payloads for '{notification.subject}'.") + logger.log_warning(message=f"Parsed failed for all the raw payloads for `{notification.subject}`.") if isinstance(raw_payload, str): raw_payload = raw_payload.encode("utf-8") @@ -282,7 +282,7 @@ def run(self, data=None, commit=None): return raw_notification_ids for notification in notifications: - self.log_info(message=f"Processing notification {notification.subject}.") + self.log_info(message=f"Processing notification `{notification.subject}`.") raw_id = process_raw_notification(self, notification) if raw_id: raw_notification_ids.append(raw_id) From e9f995b5ac45662d4866f34bc734f69990ef1222 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Wed, 21 Jul 2021 11:53:55 -0400 Subject: [PATCH 05/19] Handle whitespace in custom field; improve message formatting --- .../handle_notifications/sources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 9872bc02..f2e60a17 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -255,7 +255,9 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: """ for provider in Provider.objects.all(): if provider.custom_field_data.get("emails_circuit_maintenances"): - if email_source in provider.custom_field_data["emails_circuit_maintenances"].split(","): + if email_source in [ + source.strip() for source in provider.custom_field_data["emails_circuit_maintenances"].split(",") + ]: provider_type = provider.slug break else: @@ -267,7 +269,7 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: return ( "", "", - f"Unexpected provider {provider_type} received from {email_source}, so not getting the notification", + f"{email_source} is for provider type `{provider_type}`, not presently supported", ) return provider_data_types, provider_type, "" From f17d2a37d9d53ec98acdd536783bfebeabf31c30 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Wed, 21 Jul 2021 12:17:10 -0400 Subject: [PATCH 06/19] Fix tests --- nautobot_circuit_maintenance/tests/test_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/tests/test_sources.py b/nautobot_circuit_maintenance/tests/test_sources.py index 7e5ebf0e..73d618f5 100644 --- a/nautobot_circuit_maintenance/tests/test_sources.py +++ b/nautobot_circuit_maintenance/tests/test_sources.py @@ -128,7 +128,7 @@ def test_extract_provider_data_types_no_provider_parser(self): ( "", "", - f"Unexpected provider {provider_type} received from {email_source}, so not getting the notification", + f"{email_source} is for provider type `{provider_type}`, not presently supported", ), ) @@ -152,7 +152,7 @@ def test_extract_provider_data_types_ok(self, provider_type, data_types, error_m ( data_types if not error_message else "", provider_type if not error_message else "", - f"Unexpected provider unknown received from {email_source}, so not getting the notification" + f"{email_source} is for provider type `{provider_type}`, not presently supported" if error_message else "", ), From f90c4e571526e91b86bfc773490a369e095fb565 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Thu, 22 Jul 2021 09:18:53 -0400 Subject: [PATCH 07/19] Allow hyphens in email source matching --- nautobot_circuit_maintenance/handle_notifications/sources.py | 4 ++-- nautobot_circuit_maintenance/tests/test_sources.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index f2e60a17..09850d52 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -234,10 +234,10 @@ def validate_providers(self, job_logger: Job, notification_source: NotificationS def extract_email_source(email_source: str) -> str: """Method to get the sender email address.""" try: - email_source = re.search(r"\<([A-Za-z0-9_@.]+)\>", email_source).group(1) + email_source = re.search(r"\<([-A-Za-z0-9_@.]+)\>", email_source).group(1) except AttributeError: try: - email_source = re.search(r"([A-Za-z0-9_@.]+)", email_source).group(1) + email_source = re.search(r"([-A-Za-z0-9_@.]+)", email_source).group(1) except AttributeError: return "" return email_source diff --git a/nautobot_circuit_maintenance/tests/test_sources.py b/nautobot_circuit_maintenance/tests/test_sources.py index 73d618f5..95154a40 100644 --- a/nautobot_circuit_maintenance/tests/test_sources.py +++ b/nautobot_circuit_maintenance/tests/test_sources.py @@ -100,6 +100,7 @@ class TestEmailSource(TestCase): ["user@example.com", "user@example.com"], ["", "user@example.com"], ["user ", "user@example.com"], + ["No-reply via mailing list ", "mailing-list@example.com"], ] # pylint: disable=too-many-arguments ) def test_extract_email_source(self, email_source, email_source_output): From a83613b5d900534e53e023df4d7b8703c318ccee Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Thu, 22 Jul 2021 11:41:10 -0400 Subject: [PATCH 08/19] Address review comments, update docs, add 'source_header' config option --- README.md | 18 +- nautobot_circuit_maintenance/__init__.py | 4 +- .../handle_notifications/sources.py | 82 ++++--- .../tests/test_sources.py | 204 ++++++++++++++++-- 4 files changed, 253 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index c6e1602a..926a63bc 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Extra configuration to define notification sources is defined in the [Usage](#Us ```py PLUGINS_CONFIG = { "nautobot_circuit_maintenance": { + "source_header": "X-Original-From", # optional, see below "notification_sources": [ { ... @@ -41,23 +42,23 @@ PLUGINS_CONFIG = { } ``` -## Usage +- The `source_header` setting is used to optionally specify a particular email header to use to identify the source of a particular notification and assign it to the appropriate provider. If unset, `From` will be used, but if your emails are not received directly from the provider but instead pass through a mailing list or alias, you might need to set this to a different value such as `X-Original-From` instead. -All the plugin configuration is done via UI, under the **Plugins** tab, in the **Circuit Maintenance** sections. +## Usage ### 1. Define source emails per Provider -Each Circuit **Provider**, that we would like to track via the Circuit Maintenance plugin, requires at least one email address under the `Custom Fields` -> `Emails for Circuit Maintenance plugin` section. +In the Nautobot UI, under **Circuits -> Providers**, for each Provider that we would like to track via the Circuit Maintenance plugin, we must configure at least one email source address (or a comma-separated list of addresses) in the **`Custom Fields -> Emails for Circuit Maintenance plugin** field. -These are the source email addresses that the plugin will check and use to classify each notification for each specific provider. +These are the source email addresses that the plugin will detect and will use to classify each notification for each specific provider. ### 2. Configure Notification Sources Notification Sources are defined in two steps: -#### 2.1 Define Notification Sources in `configuration.py` +#### 2.1 Define Notification Sources in `nautobot_config.py` -In the `PLUGINS_CONFIG`, under the `nautobot_circuit_maintenance` key, we should define the Notification Sources that will be able later in the UI, where you will be able to **validate** if the authentication credentials provided are working fine or not. +In `nautobot_config.py`, in the `PLUGINS_CONFIG`, under the `nautobot_circuit_maintenance` key, we should define the Notification Sources that will be able later in the UI, where you will be able to **validate** if the authentication credentials provided are working fine or not. There are two mandatory attributes (other keys are dependent on the integration type, and will be documented below): @@ -136,12 +137,17 @@ To create a [OAuth 2.0](https://developers.google.com/identity/protocols/oauth2/ > For OAuth integration, it's recommendable that, at least the first time, you run a manual **Validate** of the Notification Source to complete the OAuth authentication workflow, identifying your Google credentials. +> Typically the `url` setting to configure in your `nautobot_config.py` for use with OAuth integration will be `"https://accounts.google.com/o/oauth2/auth"`. + + #### 2.2 Add `Providers` to the Notification Sources In the Circuit Maintenance plugin UI section, there is a **Notification Sources** button (yellow) where you can configure the Notification Sources to track new circuit maintenance notifications from specific providers. Because the Notification Sources are defined by the configuration, you can only view and edit `providers`, but not `add` or `delete` new Notification Sources via UI or API. +> Note that for emails from a given Provider to be processed, you must *both* define a source email address(es) for that Provider (Usage section 1, above) *and* add that provider to a specific Notification Source as described in this section. + ### 3. Run Handle Notifications Job There is an asynchronous task defined as a **Nautobot Job**, **Handle Circuit Maintenance Notifications** that will connect to the emails sources defined under the Notification Sources section (step above), and will fetch new notifications received since the last notification was fetched. diff --git a/nautobot_circuit_maintenance/__init__.py b/nautobot_circuit_maintenance/__init__.py index 98066317..e966c97e 100644 --- a/nautobot_circuit_maintenance/__init__.py +++ b/nautobot_circuit_maintenance/__init__.py @@ -61,7 +61,9 @@ class CircuitMaintenanceConfig(PluginConfig): min_version = "1.0.0-beta.4" max_version = "1.999" required_settings = [] - default_settings = {} + default_settings = { + "source_header": "From", + } caching_config = {} def ready(self): diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 09850d52..a0440214 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -240,7 +240,7 @@ def extract_email_source(email_source: str) -> str: email_source = re.search(r"([-A-Za-z0-9_@.]+)", email_source).group(1) except AttributeError: return "" - return email_source + return email_source.lower() @staticmethod def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: @@ -253,13 +253,16 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: str: error_message, if there was an issue ) """ + email_source = email_source.lower() for provider in Provider.objects.all(): - if provider.custom_field_data.get("emails_circuit_maintenances"): - if email_source in [ - source.strip() for source in provider.custom_field_data["emails_circuit_maintenances"].split(",") - ]: - provider_type = provider.slug - break + if not provider.custom_field_data.get("emails_circuit_maintenances"): + continue + sources = [ + src.strip().lower() for src in provider.custom_field_data["emails_circuit_maintenances"].split(",") + ] + if email_source in sources: + provider_type = provider.slug + break else: return "", "", f"Sender email {email_source} is not registered for any circuit provider." @@ -269,7 +272,7 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]: return ( "", "", - f"{email_source} is for provider type `{provider_type}`, not presently supported", + f"{email_source} is for provider type `{provider_type}`, which is not presently supported", ) return provider_data_types, provider_type, "" @@ -308,7 +311,6 @@ def _authentication_logic(self): self.open_session() self.close_session() - # pylint: disable=inconsistent-return-statements def fetch_email(self, job_logger: Job, msg_id: bytes, since: Optional[int]) -> Optional[MaintenanceNotification]: """Fetch an specific email ID.""" _, data = self.session.fetch(msg_id, "(RFC822)") @@ -318,17 +320,27 @@ def fetch_email(self, job_logger: Job, msg_id: bytes, since: Optional[int]) -> O if since: if mktime_tz(parsedate_tz(email_message["Date"])) < since: job_logger.log_info(message=f"'{email_message['Subject']}' email is old, so not taking into account.") - return + return None - email_source = self.extract_email_source(email_message["From"]) + return self.process_email(job_logger, email_message) + + def process_email( + self, job_logger: Job, email_message: email.message.EmailMessage + ) -> Optional[MaintenanceNotification]: + """Helper method for the fetch_email() method.""" + source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] + email_source = self.extract_email_source(email_message[source_header]) if not email_source: - job_logger.log_failure(message=f"Not possible to determine the email sender: {email_message['From']}") + job_logger.log_failure( + message="Not possible to determine the email sender from " + f'"{source_header}: {email_message[source_header]}"' + ) return None provider_data_types, provider_type, error_message = self.extract_provider_data_types(email_source) if not provider_data_types: job_logger.log_warning(message=error_message) - return + return None raw_payloads = [] for provider_data_type in provider_data_types: @@ -345,11 +357,11 @@ def fetch_email(self, job_logger: Job, msg_id: bytes, since: Optional[int]) -> O job_logger.log_warning( message=f"Payload types {', '.join(provider_data_types)} not found in email payload.", ) - return + return None return MaintenanceNotification( source=self.name, - sender=email_message["From"], + sender=email_message[source_header], subject=email_message["Subject"], raw_payloads=raw_payloads, provider_type=provider_type, @@ -440,50 +452,56 @@ def extract_raw_payload(self, body: Dict, msg_id: bytes) -> bytes: return b"" - # pylint: disable=inconsistent-return-statements,too-many-locals,too-many-branches, too-many-nested-blocks def fetch_email(self, job_logger: Job, msg_id: bytes, since: Optional[int]) -> Optional[MaintenanceNotification]: """Fetch an specific email ID. See data format: https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message """ - - def get_raw_payload_from_parts(parts, provider_data_type): - """Helper function to extract the raw_payload from a multiple parts via a recursive call.""" - for part_inner in parts: - for header_inner in part_inner["headers"]: - if header_inner.get("name") == "Content-Type" and provider_data_type in header_inner.get("value"): - return self.extract_raw_payload(part_inner["body"], msg_id) - if header_inner.get("name") == "Content-Type" and "multipart" in header_inner.get("value"): - return get_raw_payload_from_parts(part_inner["parts"], provider_data_type) - return None - received_email = ( self.service.users().messages().get(userId=self.account, id=msg_id).execute() # pylint: disable=no-member ) + + return self.process_email(job_logger, received_email, msg_id, since) + + def process_email( # pylint: disable=too-many-locals + self, job_logger: Job, received_email: Dict, msg_id: bytes, since: Optional[int] + ) -> Optional[MaintenanceNotification]: + """Helper method for the fetch_email() method.""" + source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] email_subject = "" email_source = "" for header in received_email["payload"]["headers"]: if header.get("name") == "Subject": email_subject = header["value"] - elif header.get("name") == "From": + elif header.get("name") == source_header: email_source = header["value"] if since: if int(received_email["internalDate"]) < since: job_logger.log_info(message=f"'{email_subject}' email is old, so not taking into account.") - return + return None email_source_before = email_source email_source = self.extract_email_source(email_source) if not email_source: job_logger.log_failure(message=f"Not possible to determine the email sender: {email_source_before}") - return + return None provider_data_types, provider_type, error_message = self.extract_provider_data_types(email_source) if not provider_data_types: job_logger.log_warning(message=error_message) - return + return None + + def get_raw_payload_from_parts(parts, provider_data_type): + """Helper function to extract the raw_payload from a multiple parts via a recursive call.""" + for part_inner in parts: + for header_inner in part_inner["headers"]: + if header_inner.get("name") == "Content-Type" and provider_data_type in header_inner.get("value"): + return self.extract_raw_payload(part_inner["body"], msg_id) + if header_inner.get("name") == "Content-Type" and "multipart" in header_inner.get("value"): + return get_raw_payload_from_parts(part_inner["parts"], provider_data_type) + return None raw_payloads = [] for provider_data_type in provider_data_types: @@ -495,7 +513,7 @@ def get_raw_payload_from_parts(parts, provider_data_type): job_logger.log_warning( message=f"Payload types {provider_data_types} not found in email payload.", ) - return + return None return MaintenanceNotification( source=self.name, diff --git a/nautobot_circuit_maintenance/tests/test_sources.py b/nautobot_circuit_maintenance/tests/test_sources.py index 95154a40..459a9f18 100644 --- a/nautobot_circuit_maintenance/tests/test_sources.py +++ b/nautobot_circuit_maintenance/tests/test_sources.py @@ -1,15 +1,22 @@ """Test sourcess utils.""" +import base64 +from email.message import EmailMessage +import json import os from unittest.mock import Mock, patch -import json +import uuid + from django.conf import settings -from django.test import TestCase +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase, override_settings from parameterized import parameterized from pydantic.error_wrappers import ValidationError # pylint: disable=no-name-in-module + from nautobot.circuits.models import Provider +from nautobot.extras.jobs import Job +from nautobot.extras.models import JobResult, Job as JobModel from nautobot_circuit_maintenance.models import NotificationSource - from nautobot_circuit_maintenance.handle_notifications.sources import ( GmailAPIOauth, GmailAPI, @@ -51,11 +58,10 @@ class TestSource(TestCase): def setUp(self): """Prepare data for tests.""" - settings.PLUGINS_CONFIG = { - "nautobot_circuit_maintenance": { - "notification_sources": [SOURCE_IMAP.copy(), SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy()] - } - } + settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["notification_sources"] = [ + SOURCE_IMAP.copy(), + SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy(), + ] # Deleting other NotificationSource to define a reliable state. NotificationSource.objects.exclude( name__in=[SOURCE_IMAP["name"], SOURCE_GMAIL_API_SERVICE_ACCOUNT["name"]] @@ -129,7 +135,7 @@ def test_extract_provider_data_types_no_provider_parser(self): ( "", "", - f"{email_source} is for provider type `{provider_type}`, not presently supported", + f"{email_source} is for provider type `{provider_type}`, which is not presently supported", ), ) @@ -153,12 +159,26 @@ def test_extract_provider_data_types_ok(self, provider_type, data_types, error_m ( data_types if not error_message else "", provider_type if not error_message else "", - f"{email_source} is for provider type `{provider_type}`, not presently supported" + f"{email_source} is for provider type `{provider_type}`, which is not presently supported" if error_message else "", ), ) + def test_extract_provider_data_types_multiple_sources(self): + """Test for extract_provider_data_types with multiple sources defined in the custom field data.""" + email_sources = ["user@example.com", "another-user@example.com"] + provider = Provider.objects.create(name="ntt", slug="ntt") + # User-inputted custom field values can be messy - we're testing that here! + provider.cf["emails_circuit_maintenances"] = " user@example.com, Another-User@example.com, " + provider.save() + + for email_source in email_sources: + self.assertEqual( + EmailSource.extract_provider_data_types(email_source), + ({"text/calendar"}, "ntt", ""), + ) + class TestIMAPSource(TestCase): """Test case for IMAP Source.""" @@ -171,7 +191,7 @@ class TestIMAPSource(TestCase): def setUp(self): """Prepare data for tests.""" - settings.PLUGINS_CONFIG = {"nautobot_circuit_maintenance": {"notification_sources": [SOURCE_IMAP.copy()]}} + settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["notification_sources"] = [SOURCE_IMAP.copy()] # Deleting other NotificationSource to define a reliable state. NotificationSource.objects.exclude(name__in=[SOURCE_IMAP["name"]]).delete() self.source = NotificationSource.objects.get(name=SOURCE_IMAP["name"]) @@ -317,6 +337,73 @@ def test_imap_init(self, name, url, account, password, imap_server, imap_port, e else: IMAP(**kwargs) + def test_process_email_success(self): + """Test successful processing of a single email into a MaintenanceNotification.""" + provider = Provider.objects.create(name="zayo", slug="zayo") + provider.cf["emails_circuit_maintenances"] = "user@example.com" + provider.save() + + source = IMAP( + name="whatever", url="imap://localhost", account="account", password="pass", imap_server="localhost" + ) + + job = Job() + job.job_result = JobResult.objects.create( + name="dummy", obj_type=ContentType.objects.get_for_model(JobModel), user=None, job_id=uuid.uuid4() + ) + + email_message = EmailMessage() + email_message["From"] = "User " + email_message["Subject"] = "Circuit Maintenance Notification" + email_message["Content-Type"] = "text/html" + email_message.set_payload("Some text goes here") + + notification = source.process_email(job, email_message) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "User ") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, "zayo") + self.assertEqual(list(notification.raw_payloads), ["Some text goes here"]) + + def test_process_email_success_alternate_source_header(self): + """Test successful processing of a single email with a non-standard source header.""" + provider = Provider.objects.create(name="zayo", slug="zayo") + provider.cf["emails_circuit_maintenances"] = "user@example.com" + provider.save() + + source = IMAP( + name="whatever", url="imap://localhost", account="account", password="pass", imap_server="localhost" + ) + + job = Job() + job.job_result = JobResult.objects.create( + name="dummy", obj_type=ContentType.objects.get_for_model(JobModel), user=None, job_id=uuid.uuid4() + ) + + email_message = EmailMessage() + email_message["From"] = "Mailing List " + email_message["X-Original-From"] = "User " + email_message["Subject"] = "Circuit Maintenance Notification" + email_message["Content-Type"] = "text/html" + email_message.set_payload("Some text goes here") + + with override_settings( + PLUGINS_CONFIG={ + "nautobot_circuit_maintenance": { + "source_header": "X-Original-From", + "notification_sources": [SOURCE_IMAP.copy()], + } + } + ): + notification = source.process_email(job, email_message) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "User ") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, "zayo") + self.assertEqual(list(notification.raw_payloads), ["Some text goes here"]) + class TestGmailAPISource(TestCase): """Test case for GmailAPI Source.""" @@ -329,11 +416,10 @@ class TestGmailAPISource(TestCase): def setUp(self): """Prepare data for tests.""" - settings.PLUGINS_CONFIG = { - "nautobot_circuit_maintenance": { - "notification_sources": [SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy(), SOURCE_GMAIL_API_OAUTH.copy()] - } - } + settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["notification_sources"] = [ + SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy(), + SOURCE_GMAIL_API_OAUTH.copy(), + ] # Deleting other NotificationSource and Provider to define a reliable state. NotificationSource.objects.exclude(name__in=[SOURCE_GMAIL_API_SERVICE_ACCOUNT["name"]]).delete() self.source = NotificationSource.objects.get(name=SOURCE_GMAIL_API_SERVICE_ACCOUNT["name"]) @@ -501,3 +587,89 @@ def test_gmail_api_test_authentication_ko(self, mock_credentials): # pylint: di res, message = source.test_authentication() self.assertEqual(res, False) self.assertEqual(message, "error message") + + @staticmethod + def email_setup(): + """Helper method for several test cases below.""" + provider = Provider.objects.create(name="zayo", slug="zayo") + provider.cf["emails_circuit_maintenances"] = "user@example.com" + provider.save() + + job = Job() + job.job_result = JobResult.objects.create( + name="dummy", obj_type=ContentType.objects.get_for_model(JobModel), user=None, job_id=uuid.uuid4() + ) + + source = GmailAPI( + name="whatever", + url="https://accounts.google.com/o/oauth2/auth", + account="account", + credentials_file="path_to_file", + ) + + return (provider, job, source) + + def test_process_email_success(self): + """Test successful processing of a single email into a MaintenanceNotification.""" + provider, job, source = self.email_setup() + + received_email = { + "payload": { + "headers": [ + {"name": "Subject", "value": "Circuit Maintenance Notification"}, + {"name": "From", "value": "user@example.com"}, + ], + "parts": [ + { + "headers": [{"name": "Content-Type", "value": "text/html"}], + "body": {"data": base64.b64encode(b"Some text goes here")}, + } + ], + }, + "internalDate": 1000, + } + + notification = source.process_email(job, received_email, msg_id="abc", since=0) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "user@example.com") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, provider.slug) + self.assertEqual(list(notification.raw_payloads), [b"Some text goes here"]) + + def test_process_email_success_alternate_source_header(self): + """Test successful processing of a single email with a non-standard source header.""" + provider, job, source = self.email_setup() + + with override_settings( + PLUGINS_CONFIG={ + "nautobot_circuit_maintenance": { + "source_header": "X-Original-From", + "notification_sources": [SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy(), SOURCE_GMAIL_API_OAUTH.copy()], + } + } + ): + received_email = { + "payload": { + "headers": [ + {"name": "Subject", "value": "Circuit Maintenance Notification"}, + {"name": "From", "value": "mailing-list@example.com"}, + {"name": "X-Original-From", "value": "user@example.com"}, + ], + "parts": [ + { + "headers": [{"name": "Content-Type", "value": "text/html"}], + "body": {"data": base64.b64encode(b"Some text goes here")}, + } + ], + }, + "internalDate": 1000, + } + + notification = source.process_email(job, received_email, msg_id="abc", since=0) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "user@example.com") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, provider.slug) + self.assertEqual(list(notification.raw_payloads), [b"Some text goes here"]) From 53888554b988531864fafe9956d4e62b280efdfe Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Thu, 22 Jul 2021 14:40:42 -0400 Subject: [PATCH 09/19] Add some debugs for troubleshooting --- nautobot_circuit_maintenance/handle_notifications/sources.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index a0440214..e077ac9b 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -515,6 +515,7 @@ def get_raw_payload_from_parts(parts, provider_data_type): ) return None + job_logger.log_debug(message=f"Got notification from {email_source} with subject {email_subject}") return MaintenanceNotification( source=self.name, sender=email_source, @@ -545,6 +546,7 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ .execute() ) msg_ids = [msg["id"] for msg in res.get("messages", [])] + job_logger.log_debug(message=f"Processing messages {msg_ids}") received_notifications = [] for msg_id in msg_ids: @@ -552,6 +554,8 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ if raw_notification: received_notifications.append(raw_notification) + job_logger.log_debug(message=f"Raw notifications: {received_notifications}") + self.close_service() return received_notifications From 7dea8930c88698d89ca2d06be5ce5c93b742f82f Mon Sep 17 00:00:00 2001 From: Uros Bajzelj Date: Thu, 22 Jul 2021 20:49:39 +0200 Subject: [PATCH 10/19] Update .travis.yml --- .travis.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea139e12..ba1c5f4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,20 +53,20 @@ jobs: - "invoke flake8" - "invoke pylint" - # - stage: "deploy-github" - # before_script: - # - "pip install poetry" - # script: - # - "poetry version $TRAVIS_TAG" - # - "poetry build" - # deploy: - # provider: "releases" - # api_key: "$GITHUB_AUTH_TOKEN" - # file_glob: true - # file: "dist/*" - # skip_cleanup: true - # "on": - # tags: true + - stage: "deploy-github" + before_script: + - "pip install poetry" + script: + - "poetry version $TRAVIS_TAG" + - "poetry build" + deploy: + provider: "releases" + api_key: "$GITHUB_AUTH_TOKEN" + file_glob: true + file: "dist/*" + skip_cleanup: true + "on": + tags: true - stage: "deploy-pypi" before_script: From 26556d00437bc26ab5d2c5e58233907daafa26cc Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 11:35:35 -0400 Subject: [PATCH 11/19] More source_header cases? --- .../handle_notifications/sources.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index e077ac9b..b899c189 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -200,7 +200,7 @@ def validate_providers(self, job_logger: Job, notification_source: NotificationS cm_cf = CustomField.objects.get(name="emails_circuit_maintenances") provider_emails = provider.get_custom_fields().get(cm_cf) if provider_emails: - self.emails_to_fetch.extend(provider_emails.split(",")) + self.emails_to_fetch.extend([src.strip().lower() for src in provider_emails.split(",")]) providers_with_email.append(provider.name) else: providers_without_email.append(provider.name) @@ -221,6 +221,7 @@ def validate_providers(self, job_logger: Job, notification_source: NotificationS ) return False + job_logger.log_debug(message=f"Fetching emails from {self.emails_to_fetch}") job_logger.log_info( message=( f"Retrieving notifications from {notification_source.name} for " @@ -384,6 +385,7 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ if self.emails_to_fetch: for sender in self.emails_to_fetch: + # TODO this needs to take configured `source_header` into account search_items = (f'FROM "{sender}"', since_date) search_text = " ".join(search_items).strip() search_criteria = f"({search_text})" @@ -535,7 +537,8 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ search_criteria = f'after:"{since_txt}"' if self.emails_to_fetch: - emails_with_from = [f"from:{email}" for email in self.emails_to_fetch] + source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] + emails_with_from = [f"{source_header}:{email}" for email in self.emails_to_fetch] search_criteria += f'({" OR ".join(emails_with_from)})' # TODO: For now not covering pagination as a way to limit the number of messages From 2b48769128456d9a990fcf8b19f484f9eed39e52 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 12:37:16 -0400 Subject: [PATCH 12/19] Gmail API doesn't allow searching on custom headers --- .../handle_notifications/sources.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index b899c189..22d4b88a 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -536,9 +536,11 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ since_txt = datetime.datetime.fromtimestamp(since).strftime("%Y/%b/%d") search_criteria = f'after:"{since_txt}"' - if self.emails_to_fetch: - source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] - emails_with_from = [f"{source_header}:{email}" for email in self.emails_to_fetch] + # If source_header is not "from" but some other custom header such as X-Original-Sender, + # the GMail API doesn't let us filter by that. + source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] + if self.emails_to_fetch and source_header.lower() == "from": + emails_with_from = [f"from:{email}" for email in self.emails_to_fetch] search_criteria += f'({" OR ".join(emails_with_from)})' # TODO: For now not covering pagination as a way to limit the number of messages From 9b4de1cf8699c24a090f6d868e313774455fddc6 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 13:39:37 -0400 Subject: [PATCH 13/19] Avoid error if payload has no parts --- .../handle_notifications/sources.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 22d4b88a..69ad30e4 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -221,7 +221,7 @@ def validate_providers(self, job_logger: Job, notification_source: NotificationS ) return False - job_logger.log_debug(message=f"Fetching emails from {self.emails_to_fetch}") + # job_logger.log_debug(message=f"Fetching emails from {self.emails_to_fetch}") job_logger.log_info( message=( f"Retrieving notifications from {notification_source.name} for " @@ -507,6 +507,11 @@ def get_raw_payload_from_parts(parts, provider_data_type): raw_payloads = [] for provider_data_type in provider_data_types: + if "parts" not in received_email["payload"]: + job_logger.log_warning( + message=f"No payload parts in message {email_subject}: {list(received_email['payload'].keys())}" + ) + continue raw_payload = get_raw_payload_from_parts(received_email["payload"]["parts"], provider_data_type) if raw_payload: raw_payloads.append(raw_payload) @@ -551,7 +556,6 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ .execute() ) msg_ids = [msg["id"] for msg in res.get("messages", [])] - job_logger.log_debug(message=f"Processing messages {msg_ids}") received_notifications = [] for msg_id in msg_ids: From 08107df405e4f239afc767eed0ffcbcd2a14d3bd Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 14:31:55 -0400 Subject: [PATCH 14/19] Handle non-multipart messages --- .../handle_notifications/sources.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 69ad30e4..3ab4d5c5 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -508,11 +508,9 @@ def get_raw_payload_from_parts(parts, provider_data_type): raw_payloads = [] for provider_data_type in provider_data_types: if "parts" not in received_email["payload"]: - job_logger.log_warning( - message=f"No payload parts in message {email_subject}: {list(received_email['payload'].keys())}" - ) - continue - raw_payload = get_raw_payload_from_parts(received_email["payload"]["parts"], provider_data_type) + raw_payload = self.extract_raw_payload(received_email["payload"]["body"], msg_id) + else: + raw_payload = get_raw_payload_from_parts(received_email["payload"]["parts"], provider_data_type) if raw_payload: raw_payloads.append(raw_payload) From b06143f60c495b40a27e95119a899b9a3c89b806 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 15:49:34 -0400 Subject: [PATCH 15/19] More message cleanup; catch more exceptions when RawNotification creation fails --- .../handle_notifications/handler.py | 6 +++--- .../handle_notifications/sources.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/handler.py b/nautobot_circuit_maintenance/handle_notifications/handler.py index 0c142ffb..524a020a 100644 --- a/nautobot_circuit_maintenance/handle_notifications/handler.py +++ b/nautobot_circuit_maintenance/handle_notifications/handler.py @@ -193,10 +193,10 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) parsed_notifications = parser.process() break except ParsingError as exc: - tb_str = traceback.format_exception(etype=type(exc), value=exc, tb=exc.__traceback__) + tb_str = traceback.format_exc() logger.log_debug(message=f"Parsing failed for notification `{notification.subject}`:\n```\n{tb_str}\n```") except Exception as exc: - tb_str = traceback.format_exception(etype=type(exc), value=exc, tb=exc.__traceback__) + tb_str = traceback.format_exc() logger.log_debug( message=f"Unexpected exception while parsing notification `{notification.subject}`.\n```\n{tb_str}\n```" ) @@ -217,7 +217,7 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) sender=notification.sender, source=NotificationSource.objects.filter(name=notification.source).last(), ) - except OperationalError as exc: + except Exception as exc: logger.log_warning(message=f"Raw notification '{notification.subject}' not created because {str(exc)}") return None diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index 3ab4d5c5..dad6039a 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -332,7 +332,7 @@ def process_email( source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] email_source = self.extract_email_source(email_message[source_header]) if not email_source: - job_logger.log_failure( + job_logger.log_warning( message="Not possible to determine the email sender from " f'"{source_header}: {email_message[source_header]}"' ) @@ -487,7 +487,7 @@ def process_email( # pylint: disable=too-many-locals email_source_before = email_source email_source = self.extract_email_source(email_source) if not email_source: - job_logger.log_failure(message=f"Not possible to determine the email sender: {email_source_before}") + job_logger.log_warning(message=f'Not possible to determine the email sender from "{email_source_before}"') return None provider_data_types, provider_type, error_message = self.extract_provider_data_types(email_source) @@ -561,7 +561,7 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ if raw_notification: received_notifications.append(raw_notification) - job_logger.log_debug(message=f"Raw notifications: {received_notifications}") + # job_logger.log_debug(message=f"Raw notifications: {received_notifications}") self.close_service() return received_notifications From 680460dc28b5db1318d1da1a004f3a3a190fde47 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 16:28:25 -0400 Subject: [PATCH 16/19] Lint fixes --- nautobot_circuit_maintenance/handle_notifications/handler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/handler.py b/nautobot_circuit_maintenance/handle_notifications/handler.py index 524a020a..2f56095b 100644 --- a/nautobot_circuit_maintenance/handle_notifications/handler.py +++ b/nautobot_circuit_maintenance/handle_notifications/handler.py @@ -3,7 +3,6 @@ import traceback from typing import Union from django.conf import settings -from django.db import OperationalError from circuit_maintenance_parser import ParsingError, init_provider from nautobot.circuits.models import Circuit, Provider from nautobot.extras.jobs import Job @@ -192,10 +191,10 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) try: parsed_notifications = parser.process() break - except ParsingError as exc: + except ParsingError: tb_str = traceback.format_exc() logger.log_debug(message=f"Parsing failed for notification `{notification.subject}`:\n```\n{tb_str}\n```") - except Exception as exc: + except Exception: tb_str = traceback.format_exc() logger.log_debug( message=f"Unexpected exception while parsing notification `{notification.subject}`.\n```\n{tb_str}\n```" From b6ba6b812dee5ca1ce5839f3a5a1f59a7359921a Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Fri, 23 Jul 2021 16:44:07 -0400 Subject: [PATCH 17/19] Don't try to create a RawNotification if there's no valid raw_payload --- nautobot_circuit_maintenance/handle_notifications/handler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nautobot_circuit_maintenance/handle_notifications/handler.py b/nautobot_circuit_maintenance/handle_notifications/handler.py index 2f56095b..f2294666 100644 --- a/nautobot_circuit_maintenance/handle_notifications/handler.py +++ b/nautobot_circuit_maintenance/handle_notifications/handler.py @@ -182,6 +182,7 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) ) return None + raw_payload = b"" for raw_payload in notification.raw_payloads: parser = init_provider(raw=raw_payload, provider_type=notification.provider_type) if not parser: @@ -201,8 +202,10 @@ def process_raw_notification(logger: Job, notification: MaintenanceNotification) ) else: parsed_notifications = [] - raw_payload = b"" logger.log_warning(message=f"Parsed failed for all the raw payloads for `{notification.subject}`.") + # Carry on with the last raw_payload in the list, if any + if not raw_payload: + return None if isinstance(raw_payload, str): raw_payload = raw_payload.encode("utf-8") From 22a49bce81a13923e900a2f51bdd24568c17fc58 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Mon, 26 Jul 2021 10:41:55 -0400 Subject: [PATCH 18/19] Change source_header to a per-notification-source config property --- README.md | 43 +++++---- nautobot_circuit_maintenance/__init__.py | 4 +- .../handle_notifications/sources.py | 22 +++-- .../tests/test_sources.py | 91 +++++++++---------- 4 files changed, 79 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 926a63bc..137e9ec3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ Extra configuration to define notification sources is defined in the [Usage](#Us ```py PLUGINS_CONFIG = { "nautobot_circuit_maintenance": { - "source_header": "X-Original-From", # optional, see below "notification_sources": [ { ... @@ -42,8 +41,6 @@ PLUGINS_CONFIG = { } ``` -- The `source_header` setting is used to optionally specify a particular email header to use to identify the source of a particular notification and assign it to the appropriate provider. If unset, `From` will be used, but if your emails are not received directly from the provider but instead pass through a mailing list or alias, you might need to set this to a different value such as `X-Original-From` instead. - ## Usage ### 1. Define source emails per Provider @@ -67,36 +64,45 @@ There are two mandatory attributes (other keys are dependent on the integration > Currently, only IMAP and HTTPS (accounts.google.com) integrations are supported as URL scheme -##### IMAP +##### 2.1.1 IMAP -There are 2 extra attributes: +There are 2 extra required attributes: - `account`: Identifier (i.e. email address) to use to authenticate. - `secret`: Password to IMAP authentication. > Gmail example: [How to setup Gmail with App Passwords](https://support.google.com/accounts/answer/185833) +There is also one optional attribute: + +- `source_header`: Specify a particular email header to use to identify the source of a particular notification and assign it to the appropriate provider. If unset, `From` will be used, but if your emails are not received directly from the provider but instead pass through a mailing list or alias, you might need to set this to a different value such as `X-Original-Sender` instead. + ```py PLUGINS_CONFIG = { "nautobot_circuit_maintenance": { "notification_sources": [ { "name": "my custom name", - "account": os.environ.get("CM_NS_1_ACCOUNT", ""), - "secret": os.environ.get("CM_NS_1_SECRET", ""), - "url": os.environ.get("CM_NS_1_URL", ""), + "account": os.getenv("CM_NS_1_ACCOUNT", ""), + "secret": os.getenv("CM_NS_1_SECRET", ""), + "url": os.getenv("CM_NS_1_URL", ""), + "source_header": os.getenv("CM_NS_1_SOURCE_HEADER", "From"), # optional } ] } } ``` -##### Gmail API integrations +##### 2.1.2 Gmail API integrations + +There are 2 extra required attributes: + +- `account`: Identifier (i.e. email address) to access via OAuth or to impersonate as service account. +- `credentials_file`: JSON file containing all the necessary data to identify the API integration (see below). -There are 2 extra attributes: +There is also one optional attribute: -- `account`: Identifier (i.e. email address) to use to impersonate as service account. -- `credentials_file`: JSON file containing all the necessary data to identify the API integration. +- `source_header`: Specify a particular email header to use to identify the source of a particular notification and assign it to the appropriate provider. If unset, `From` will be used, but if your emails are not received directly from the provider but instead pass through a mailing list or alias, you might need to set this to a different value such as `X-Original-Sender` instead. ```py PLUGINS_CONFIG = { @@ -104,21 +110,22 @@ PLUGINS_CONFIG = { "notification_sources": [ { "name": "my custom name", - "account": os.environ.get("CM_NS_1_ACCOUNT", ""), - "credentials_file": os.environ.get("CM_NS_1_CREDENTIALS_FILE", ""), - "url": os.environ.get("CM_NS_1_URL", ""), + "account": os.getenv("CM_NS_1_ACCOUNT", ""), + "credentials_file": os.getenv("CM_NS_1_CREDENTIALS_FILE", ""), + "url": os.getenv("CM_NS_1_URL", ""), + "source_header": os.getenv("CM_NS_1_SOURCE_HEADER", "From"), # optional } ] } } ``` -To enable Gmail API access, there are some common steps for either Service Account and OAuth authentication: +To enable Gmail API access, there are some common steps for both Service Account and OAuth authentication: 1. Create a **New Project** in [Google Cloud Console](https://console.cloud.google.com/). 2. Under **APIs and Services**, enable **Gmail API** for the selected project. -**Service Account** +###### 2.1.2.1 Service Account To create a [Service Account](https://support.google.com/a/answer/7378726?hl=en) integration: @@ -127,7 +134,7 @@ To create a [Service Account](https://support.google.com/a/answer/7378726?hl=en) 5. With Super Admin rights, open the [Google Workspace admin console](https://admin.google.com). Navigate to **Security**, **API controls**, and select the **Manage Domain Wide Delegation** at the bottom of the page. 6. Add a new API client and paste in the Client ID copied earlier. In the **OAuth scopes** field add the scopes `https://www.googleapis.com/auth/gmail.readonly` and `https://mail.google.com/`. Save the new client configuration by clicking _Authorize_. -**OAuth** +###### 2.1.2.2 OAuth To create a [OAuth 2.0](https://developers.google.com/identity/protocols/oauth2/web-server) integration: diff --git a/nautobot_circuit_maintenance/__init__.py b/nautobot_circuit_maintenance/__init__.py index e966c97e..98066317 100644 --- a/nautobot_circuit_maintenance/__init__.py +++ b/nautobot_circuit_maintenance/__init__.py @@ -61,9 +61,7 @@ class CircuitMaintenanceConfig(PluginConfig): min_version = "1.0.0-beta.4" max_version = "1.999" required_settings = [] - default_settings = { - "source_header": "From", - } + default_settings = {} caching_config = {} def ready(self): diff --git a/nautobot_circuit_maintenance/handle_notifications/sources.py b/nautobot_circuit_maintenance/handle_notifications/sources.py index dad6039a..59bfe8b0 100644 --- a/nautobot_circuit_maintenance/handle_notifications/sources.py +++ b/nautobot_circuit_maintenance/handle_notifications/sources.py @@ -138,6 +138,7 @@ def init(cls: Type[T], name: str) -> T: password=config.get("secret"), imap_server=url_components.netloc.split(":")[0], imap_port=url_components.port or 993, + source_header=config.get("source_header", "From"), ) if scheme == "https" and url_components.netloc.split(":")[0] == "accounts.google.com": creds_filename = config.get("credentials_file") @@ -160,6 +161,7 @@ def init(cls: Type[T], name: str) -> T: url=url, account=config.get("account"), credentials_file=creds_filename, + source_header=config.get("source_header", "From"), ) raise ValueError( @@ -172,6 +174,7 @@ class EmailSource(Source): # pylint: disable=abstract-method account: str emails_to_fetch = [] + source_header: str = "From" def get_account_id(self) -> str: """Method to get an identifier of the related account.""" @@ -329,12 +332,11 @@ def process_email( self, job_logger: Job, email_message: email.message.EmailMessage ) -> Optional[MaintenanceNotification]: """Helper method for the fetch_email() method.""" - source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] - email_source = self.extract_email_source(email_message[source_header]) + email_source = self.extract_email_source(email_message[self.source_header]) if not email_source: job_logger.log_warning( message="Not possible to determine the email sender from " - f'"{source_header}: {email_message[source_header]}"' + f'"{self.source_header}: {email_message[self.source_header]}"' ) return None @@ -362,7 +364,7 @@ def process_email( return MaintenanceNotification( source=self.name, - sender=email_message[source_header], + sender=email_message[self.source_header], subject=email_message["Subject"], raw_payloads=raw_payloads, provider_type=provider_type, @@ -385,8 +387,10 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ if self.emails_to_fetch: for sender in self.emails_to_fetch: - # TODO this needs to take configured `source_header` into account - search_items = (f'FROM "{sender}"', since_date) + if self.source_header == "From": + search_items = (f'FROM "{sender}"', since_date) + else: + search_items = (f'HEADER {self.source_header} "{sender}"', since_date) search_text = " ".join(search_items).strip() search_criteria = f"({search_text})" messages = self.session.search(None, search_criteria)[1][0] @@ -469,14 +473,13 @@ def process_email( # pylint: disable=too-many-locals self, job_logger: Job, received_email: Dict, msg_id: bytes, since: Optional[int] ) -> Optional[MaintenanceNotification]: """Helper method for the fetch_email() method.""" - source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] email_subject = "" email_source = "" for header in received_email["payload"]["headers"]: if header.get("name") == "Subject": email_subject = header["value"] - elif header.get("name") == source_header: + elif header.get("name") == self.source_header: email_source = header["value"] if since: @@ -541,8 +544,7 @@ def receive_notifications(self, job_logger: Job, since: int = None) -> Iterable[ # If source_header is not "from" but some other custom header such as X-Original-Sender, # the GMail API doesn't let us filter by that. - source_header = settings.PLUGINS_CONFIG["nautobot_circuit_maintenance"]["source_header"] - if self.emails_to_fetch and source_header.lower() == "from": + if self.emails_to_fetch and self.source_header == "From": emails_with_from = [f"from:{email}" for email in self.emails_to_fetch] search_criteria += f'({" OR ".join(emails_with_from)})' diff --git a/nautobot_circuit_maintenance/tests/test_sources.py b/nautobot_circuit_maintenance/tests/test_sources.py index 459a9f18..0fbc110e 100644 --- a/nautobot_circuit_maintenance/tests/test_sources.py +++ b/nautobot_circuit_maintenance/tests/test_sources.py @@ -8,7 +8,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.test import TestCase, override_settings +from django.test import TestCase from parameterized import parameterized from pydantic.error_wrappers import ValidationError # pylint: disable=no-name-in-module @@ -373,7 +373,12 @@ def test_process_email_success_alternate_source_header(self): provider.save() source = IMAP( - name="whatever", url="imap://localhost", account="account", password="pass", imap_server="localhost" + name="whatever", + url="imap://localhost", + account="account", + password="pass", + imap_server="localhost", + source_header="X-Original-Sender", ) job = Job() @@ -383,26 +388,18 @@ def test_process_email_success_alternate_source_header(self): email_message = EmailMessage() email_message["From"] = "Mailing List " - email_message["X-Original-From"] = "User " + email_message["X-Original-Sender"] = "User " email_message["Subject"] = "Circuit Maintenance Notification" email_message["Content-Type"] = "text/html" email_message.set_payload("Some text goes here") - with override_settings( - PLUGINS_CONFIG={ - "nautobot_circuit_maintenance": { - "source_header": "X-Original-From", - "notification_sources": [SOURCE_IMAP.copy()], - } - } - ): - notification = source.process_email(job, email_message) - self.assertIsNotNone(notification) - self.assertEqual(notification.source, source.name) - self.assertEqual(notification.sender, "User ") - self.assertEqual(notification.subject, "Circuit Maintenance Notification") - self.assertEqual(notification.provider_type, "zayo") - self.assertEqual(list(notification.raw_payloads), ["Some text goes here"]) + notification = source.process_email(job, email_message) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "User ") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, "zayo") + self.assertEqual(list(notification.raw_payloads), ["Some text goes here"]) class TestGmailAPISource(TestCase): @@ -641,35 +638,29 @@ def test_process_email_success_alternate_source_header(self): """Test successful processing of a single email with a non-standard source header.""" provider, job, source = self.email_setup() - with override_settings( - PLUGINS_CONFIG={ - "nautobot_circuit_maintenance": { - "source_header": "X-Original-From", - "notification_sources": [SOURCE_GMAIL_API_SERVICE_ACCOUNT.copy(), SOURCE_GMAIL_API_OAUTH.copy()], - } - } - ): - received_email = { - "payload": { - "headers": [ - {"name": "Subject", "value": "Circuit Maintenance Notification"}, - {"name": "From", "value": "mailing-list@example.com"}, - {"name": "X-Original-From", "value": "user@example.com"}, - ], - "parts": [ - { - "headers": [{"name": "Content-Type", "value": "text/html"}], - "body": {"data": base64.b64encode(b"Some text goes here")}, - } - ], - }, - "internalDate": 1000, - } - - notification = source.process_email(job, received_email, msg_id="abc", since=0) - self.assertIsNotNone(notification) - self.assertEqual(notification.source, source.name) - self.assertEqual(notification.sender, "user@example.com") - self.assertEqual(notification.subject, "Circuit Maintenance Notification") - self.assertEqual(notification.provider_type, provider.slug) - self.assertEqual(list(notification.raw_payloads), [b"Some text goes here"]) + source.source_header = "X-Original-Sender" + + received_email = { + "payload": { + "headers": [ + {"name": "Subject", "value": "Circuit Maintenance Notification"}, + {"name": "From", "value": "mailing-list@example.com"}, + {"name": "X-Original-Sender", "value": "user@example.com"}, + ], + "parts": [ + { + "headers": [{"name": "Content-Type", "value": "text/html"}], + "body": {"data": base64.b64encode(b"Some text goes here")}, + } + ], + }, + "internalDate": 1000, + } + + notification = source.process_email(job, received_email, msg_id="abc", since=0) + self.assertIsNotNone(notification) + self.assertEqual(notification.source, source.name) + self.assertEqual(notification.sender, "user@example.com") + self.assertEqual(notification.subject, "Circuit Maintenance Notification") + self.assertEqual(notification.provider_type, provider.slug) + self.assertEqual(list(notification.raw_payloads), [b"Some text goes here"]) From facbdb9ffa1c35c5f5f0f6a17b48905f412157a3 Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Mon, 26 Jul 2021 16:32:22 -0400 Subject: [PATCH 19/19] Release v0.1.7 --- CHANGELOG.md | 17 +++++++++++++++++ nautobot_circuit_maintenance/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8e6da02..fc41c8fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## v0.1.7 - 2021-07-27 + +### Added + +- #42: + - Add stack trace to job log on exception + - IMAP and GMail notification sources now support a `source_header` configuration parameter to allow for cases where `From` is not the relevant header to inspect. + + +### Fixed + +- #42: + - Avoid an exception if some Providers do not have a populated `emails_circuit_maintenance` value + - `extract_email_source()` now correctly handles email addresses containing dash characters. + - Avoid an exception on processing a non-multipart email payload + - Don't try to create a `RawNotification` if no `raw_payload` could be extracted from the notification. + ## v0.1.6 - 2021-07-14 ### Added diff --git a/nautobot_circuit_maintenance/__init__.py b/nautobot_circuit_maintenance/__init__.py index 98066317..06593a00 100644 --- a/nautobot_circuit_maintenance/__init__.py +++ b/nautobot_circuit_maintenance/__init__.py @@ -1,5 +1,5 @@ """Init for Circuit Maintenance plugin.""" -__version__ = "0.1.6" +__version__ = "0.1.7" from django.conf import settings from django.db.models.signals import post_migrate from django.utils.text import slugify diff --git a/pyproject.toml b/pyproject.toml index aeca2296..77213e18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nautobot-circuit-maintenance" -version = "0.1.6" +version = "0.1.7" description = "Nautobot plugin to automatically handle Circuit Maintenances Notifications" authors = ["Network to Code, LLC "]