Fix missing petitions when importing form Portal

dear_petition/petition/conftest.py

@@ -173,7 +173,7 @@ def client():
 @pytest.fixture
 def dismissed_offense(record1):
     return OffenseFactory(
-        disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+        disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
         ciprs_record=record1,
         jurisdiction=constants.DISTRICT_COURT,
     )
@@ -209,18 +209,6 @@ def convicted_guilty_record(guilty_offense):
     yield OffenseRecordFactory(action=CONVICTED, offense=guilty_offense)
 
 
-@pytest.fixture
-def adult_convicted_guilty_record(record1):
-    guilty_offense = OffenseFactory(
-        ciprs_record=record1,
-        jurisdiction=constants.DISTRICT_COURT,
-        verdict="GUILTY",
-        disposition_method="",
-        disposed_on=date(2010, 1, 1), # account for waiting period
-    )
-    yield OffenseRecordFactory(action=CONVICTED, offense=guilty_offense)
-
-
 @pytest.fixture
 def not_guilty_offense(record1):
     return OffenseFactory(

dear_petition/petition/constants.py

@@ -98,7 +98,7 @@
 
 DISTRICT_COURT_WITHOUT_DA_LEAVE = "Dismissal without Leave by DA"
 
-DISMISSED_DISPOSITION_METHODS = (
+CIPRS_DISPOSITION_METHODS_DISMISSED = (
     DISTRICT_COURT_WITHOUT_DA_LEAVE,
     "Dismissed by Court",
     "Deferred Prosecution Dismissal",
@@ -107,9 +107,22 @@
     "No Probable Cause",
     "Never To Be Served",
     "Deferred Proceeding or Deferred Prosecution Dismissal",
-    # Portal:
+)
+
+PORTAL_DISPOSITION_METHODS_DISMISSED = (
     "VD-District Dismissals w/o Leave by DA - No Plea Agreement",
     "VD-Superior Dismissals w/o Leave by DA - No Plea Agreement",
+    "No Probable Cause Found",
+)
+
+PORTAL_DISPOSITION_METHODS_NOT_GUILTY = (
+    "District Not Guilty - Judge",
+    "Superior Not Guilty - Judge"
+)
+
+PORTAL_DISPOSITION_METHODS_CONVICTED = (
+    "District Guilty - Judge",
+    "Superior Guilty - Judge"
 )
 
 DISP_METHOD_SUPERSEDING_INDICTMENT = "SUPERSEDING INDICTMENT OR PROCESS"
@@ -183,5 +196,5 @@
 
 SEVERITY_FELONY = "FELONY"
 SEVERITY_MISDEMEANOR = "MISDEMEANOR"
-CHARGED_DEGREE_FELONY = ("FH", "FNC")
-CHARGED_DEGREE_MISDEMEANOR = ("MNC",)
+CHARGED_DEGREE_FELONY = ("FA", "FB", "FC", "FD", "FE", "FF", "FG", "FH", "FI", "FNC")
+CHARGED_DEGREE_MISDEMEANOR = ("MA1", "M1", "M2", "M3", "MNC")

dear_petition/petition/etl/tests/test_petition_offenses.py

@@ -20,7 +20,7 @@ def many_offense_records(batch, size):
             batch=batch, jurisdiction=constants.DISTRICT_COURT, county="DURHAM"
         )
         offense = OffenseFactory(
-            disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+            disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
             ciprs_record=record,
             jurisdiction=constants.DISTRICT_COURT,
         )
@@ -146,7 +146,7 @@ def test_paginator_same_record_number_order(petition, records_10):
     # attach a 2nd dismissed charge
     charge_2 = OffenseRecordFactory(
         offense=OffenseFactory(
-            disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+            disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
             ciprs_record=charge_1.offense.ciprs_record,
             jurisdiction=constants.DISTRICT_COURT,
         ),
@@ -176,7 +176,7 @@ def create_new_ciprs_record(file_no):
         offense = OffenseFactory(
             ciprs_record=ciprs_record,
             jurisdiction=constants.DISTRICT_COURT,
-            disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+            disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
         )
         offense_record = OffenseRecordFactory(offense=offense)
         return offense_record.id

dear_petition/petition/types/adult_felonies.py

@@ -4,6 +4,7 @@
 from django.utils import timezone
 from django.db.models import Q
 
+from dear_petition.petition.constants import PORTAL_DISPOSITION_METHODS_CONVICTED
 from dear_petition.petition.models import OffenseRecord
 from dear_petition.petition import constants as pc
 
@@ -25,5 +26,12 @@ def build_query():
     today = timezone.now().date()
     waiting_period_start_date = today - relativedelta(years=10)
     waiting_period = Q(offense__disposed_on__lt=waiting_period_start_date)
-    query = action & severity & waiting_period
-    return query
+    adult_felony_ciprs = action & severity & waiting_period
+
+    methods = Q()
+    for method in PORTAL_DISPOSITION_METHODS_CONVICTED:
+        methods |= Q(offense__disposition_method__iexact=method)
+    adult_felony_portal = methods & severity & waiting_period
+
+    return adult_felony_ciprs | adult_felony_portal
+

dear_petition/petition/types/adult_misdemeanors.py

@@ -4,6 +4,7 @@
 from django.db.models import Q
 from django.utils import timezone
 
+from dear_petition.petition.constants import PORTAL_DISPOSITION_METHODS_CONVICTED
 from dear_petition.petition.models import OffenseRecord
 from dear_petition.petition import constants as pc
 
@@ -25,5 +26,11 @@ def build_query():
     today = timezone.now().date()
     waiting_period_start_date = today - relativedelta(years=5)
     waiting_period = Q(offense__disposed_on__lt=waiting_period_start_date)
-    query = action & severity & waiting_period
-    return query
+    adult_misdemeanor_ciprs = action & severity & waiting_period
+
+    methods = Q()
+    for method in PORTAL_DISPOSITION_METHODS_CONVICTED:
+        methods |= Q(offense__disposition_method__iexact=method)
+    adult_misdemeanor_portal = methods & severity & waiting_period
+
+    return adult_misdemeanor_ciprs | adult_misdemeanor_portal

dear_petition/petition/types/dismissed.py

@@ -1,7 +1,7 @@
 from django.db.models import Q
 
 from dear_petition.petition.models import OffenseRecord
-from dear_petition.petition.constants import DISMISSED_DISPOSITION_METHODS
+from dear_petition.petition.constants import CIPRS_DISPOSITION_METHODS_DISMISSED, PORTAL_DISPOSITION_METHODS_DISMISSED
 
 
 def get_offense_records(batch, jurisdiction=""):
@@ -14,8 +14,15 @@ def get_offense_records(batch, jurisdiction=""):
 
 def build_query():
     action = Q(action="CHARGED")
-    methods = Q()
-    for method in DISMISSED_DISPOSITION_METHODS:
-        methods |= Q(offense__disposition_method__iexact=method)
-    query = action & methods
-    return query
+
+    methods_ciprs = Q()
+    for method in CIPRS_DISPOSITION_METHODS_DISMISSED:
+        methods_ciprs |= Q(offense__disposition_method__iexact=method)
+    dismissed_ciprs = action & methods_ciprs
+
+    methods_portal = Q()
+    for method in PORTAL_DISPOSITION_METHODS_DISMISSED:
+        methods_portal |= Q(offense__disposition_method__iexact=method)
+    dismissed_portal = methods_portal
+
+    return dismissed_ciprs | dismissed_portal

dear_petition/petition/types/not_guilty.py

@@ -1,5 +1,6 @@
 from django.db.models import Q
 
+from dear_petition.petition.constants import PORTAL_DISPOSITION_METHODS_NOT_GUILTY
 from dear_petition.petition.models import OffenseRecord
 from dear_petition.petition.types.dismissed import build_query as build_dismissed_query
 
@@ -17,5 +18,11 @@ def build_query():
     action = Q(action="CHARGED")
     verdict = Q(offense__verdict__iexact="Not Guilty")
     dismissed_query = build_dismissed_query()
-    query = action & verdict & ~dismissed_query
-    return query
+    not_guilty_ciprs = action & verdict & ~dismissed_query
+
+    methods = Q()
+    for method in PORTAL_DISPOSITION_METHODS_NOT_GUILTY:
+        methods |= Q(offense__disposition_method__iexact=method)
+    not_guilty_portal = methods
+
+    return not_guilty_ciprs | not_guilty_portal

dear_petition/petition/types/tests/test_adult_felonies.py

@@ -1,19 +1,40 @@
+from datetime import datetime
+
 import pytest
 
-from dear_petition.petition import constants
+from dear_petition.petition.constants import CONVICTED, CHARGED, SEVERITIES
+from dear_petition.petition.models import Offense, OffenseRecord
 
 pytestmark = pytest.mark.django_db
 
 
-def test_adult_felony_included(batch, adult_convicted_guilty_record):
-    adult_convicted_guilty_record.severity = constants.SEVERITIES.FELONY
-    adult_convicted_guilty_record.save()
-
-    assert adult_convicted_guilty_record in batch.adult_felony_records()
-
-
-def test_adult_misdemeanor_not_included(batch, adult_convicted_guilty_record):
-    adult_convicted_guilty_record.severity = constants.SEVERITIES.MISDEMEANOR
-    adult_convicted_guilty_record.save()
-
-    assert adult_convicted_guilty_record not in batch.adult_felony_records()
+@pytest.mark.parametrize(
+    "action, disposition_method, severity, date, should_be_included", [
+        # records that have data as they would from Portal (no action)
+        ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), True),
+        ("", "District Not Guilty - Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), False),  # exclude because not Portal guilty disposition method
+        ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2014, 8, 1), False),  # exclude because misdemeanor severity
+        ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2024, 8, 1), False),  # exclude because too recent
+        # records that have data as they would from CIPRS (disposition_method not one seen in Portal)
+        (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), True),
+        (CHARGED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), False),  # exclude because not convicted action
+        (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2014, 8, 1), False),  # exclude because misdemeanor severity
+        (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2024, 8, 1), False),  # exclude because too recent
+    ]
+)
+def test_adult_felonies(action, disposition_method, severity, date, should_be_included, batch, record1):
+    offense = Offense.objects.create(
+        ciprs_record=record1,
+        disposition_method=disposition_method,
+        disposed_on=date,
+    )
+    offense_record = OffenseRecord.objects.create(
+        offense=offense,
+        action=action,
+        severity=severity,
+    )
+
+    if should_be_included:
+        assert offense_record in batch.adult_felony_records()
+    else:
+        assert offense_record not in batch.adult_felony_records()

dear_petition/petition/types/tests/test_adult_misdemeanors.py

@@ -1,19 +1,40 @@
+from datetime import datetime
+
 import pytest
 
-from dear_petition.petition import constants
+from dear_petition.petition.constants import SEVERITIES, CONVICTED, CHARGED
+from dear_petition.petition.models import Offense, OffenseRecord
 
 pytestmark = pytest.mark.django_db
 
 
-def test_adult_misdemeanor_included(batch, adult_convicted_guilty_record):
-    adult_convicted_guilty_record.severity = constants.SEVERITIES.MISDEMEANOR
-    adult_convicted_guilty_record.save()
-
-    assert adult_convicted_guilty_record in batch.adult_misdemeanor_records()
-
-
-def test_adult_felony_not_included(batch, adult_convicted_guilty_record):
-    adult_convicted_guilty_record.severity = constants.SEVERITIES.FELONY
-    adult_convicted_guilty_record.save()
-
-    assert adult_convicted_guilty_record not in batch.adult_misdemeanor_records()
+@pytest.mark.parametrize(
+    "action, disposition_method, severity, date, should_be_included", [
+        # records that have data as they would from Portal (no action)
+        ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), True),
+        ("", "District Not Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), False),  # exclude because not Portal guilty disposition method
+        ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2019, 8, 1), False),  # exclude because felony severity
+        ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2024, 8, 1), False),  # exclude because too recent
+        # records that have data as they would from CIPRS (disposition_method not one seen in Portal)
+        (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), True),
+        (CHARGED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), False),  # exclude because not convicted action
+        (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2019, 8, 1), False),  # exclude because felony severity
+        (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2024, 8, 1), False),  # exclude because too recent
+    ]
+)
+def test_adult_misdemeanors(action, disposition_method, severity, date, should_be_included, batch, record1):
+    offense = Offense.objects.create(
+        ciprs_record=record1,
+        disposition_method=disposition_method,
+        disposed_on=date,
+    )
+    offense_record = OffenseRecord.objects.create(
+        offense=offense,
+        action=action,
+        severity=severity,
+    )
+
+    if should_be_included:
+        assert offense_record in batch.adult_misdemeanor_records()
+    else:
+        assert offense_record not in batch.adult_misdemeanor_records()

dear_petition/petition/types/tests/test_dismissed.py

@@ -7,30 +7,37 @@
     OffenseRecordFactory,
     PetitionFactory,
 )
+from dear_petition.petition.constants import CONVICTED, CHARGED
+from dear_petition.petition.models import Offense, OffenseRecord
 
 pytestmark = pytest.mark.django_db
 
 
-@pytest.mark.parametrize("method", constants.DISMISSED_DISPOSITION_METHODS)
-def test_charged_disposition_methods(batch, record1, method):
-    """CHARGED offense records should be included for all dismissed disposition methods."""
-    offense = OffenseFactory(disposition_method=method, ciprs_record=record1)
-    offense_record = OffenseRecordFactory(action="CHARGED", offense=offense)
-    assert offense_record in batch.dismissed_offense_records()
-
-
-def test_non_charged_offense_record(batch, dismissed_offense):
-    """Non-CHARGED dismissed offense records should be excluded."""
-    offense_record = OffenseRecordFactory(action="CONVICTED", offense=dismissed_offense)
-    assert offense_record not in batch.dismissed_offense_records()
-
-
-def test_non_dismissed_disposition_method(batch, non_dismissed_offense):
-    """Offenses with non-dismissed disposition methods should be excluded."""
-    offense_record = OffenseRecordFactory(
-        action="CHARGED", offense=non_dismissed_offense
+@pytest.mark.parametrize(
+    "action, disposition_method, should_be_included", [
+        # records that have data as they would from Portal (no action)
+        ("", "No Probable Cause Found", True),
+        ("", "District Guilty - Judge", False),  # exclude because not Portal dismissed disposition method
+        # records that have data as they would from CIPRS (disposition_method not one seen in Portal)
+        (CHARGED, "No Probable Cause", True),
+        (CONVICTED, "No Probable Cause", False),  # exclude because not charged action
+        (CHARGED, "Disposed By Judge", False),  # exclude because not CIPRS dismissed disposition method
+    ]
+)
+def test_dismissed(action, disposition_method, should_be_included, batch, record1):
+    offense = Offense.objects.create(
+        ciprs_record=record1,
+        disposition_method=disposition_method,
     )
-    assert offense_record not in batch.dismissed_offense_records()
+    offense_record = OffenseRecord.objects.create(
+        offense=offense,
+        action=action,
+    )
+
+    if should_be_included:
+        assert offense_record in batch.dismissed_offense_records()
+    else:
+        assert offense_record not in batch.dismissed_offense_records()
 
 
 def test_infraction_severity_offense_record(batch, dismissed_offense):
@@ -52,7 +59,7 @@ def test_offense_records_by_jurisdiction(batch, jurisdiction):
     """Offense records helper function should allow filtering by jurisdiction."""
     ciprs_record = CIPRSRecordFactory(jurisdiction=jurisdiction, batch=batch)
     offense = OffenseFactory(
-        disposition_method=constants.DISMISSED_DISPOSITION_METHODS[0],
+        disposition_method=constants.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
         ciprs_record=ciprs_record,
     )
     offense_record = OffenseRecordFactory(action="CHARGED", offense=offense)

dear_petition/petition/types/tests/test_distinct_petitions.py

@@ -24,7 +24,7 @@ def test_distinct_petition(batch, dismissed_offense):
 
 def test_distinct_petition__many(batch):
     """Identified petitions should include unique pairing of jurisdiction and county."""
-    method = constants.DISMISSED_DISPOSITION_METHODS[0]
+    method = constants.CIPRS_DISPOSITION_METHODS_DISMISSED[0]
     for jurisdiction in [constants.DISTRICT_COURT, constants.SUPERIOR_COURT]:
         for county in ["DURHAM", "WAKE"]:
             record = CIPRSRecordFactory(
@@ -50,7 +50,7 @@ def test_distinct_petition__distinct_ciprs_records(batch):
     OffenseRecordFactory(
         action="CHARGED",
         offense=OffenseFactory(
-            disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+            disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
             ciprs_record=record1,
         ),
     )
@@ -63,7 +63,7 @@ def test_distinct_petition__distinct_ciprs_records(batch):
     OffenseRecordFactory(
         action="CHARGED",
         offense=OffenseFactory(
-            disposition_method=dismissed.DISMISSED_DISPOSITION_METHODS[0],
+            disposition_method=dismissed.CIPRS_DISPOSITION_METHODS_DISMISSED[0],
             ciprs_record=record2,
         ),
     )