Added counter field and updated confirmation_id

db_revisions/versions/6ec12afa5b37_add_counter_to_submission_table.py

@@ -0,0 +1,52 @@
+"""add counter to submission table
+
+Revision ID: 6ec12afa5b37
+Revises: 26f11ac15b3c
+Create Date: 2024-10-28 10:52:22.353469
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "6ec12afa5b37"
+down_revision: Union[str, None] = "26f11ac15b3c"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # create the new column as nullable so it doesn't error with existing data
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("counter", sa.Integer, nullable=True))
+        batch_op.create_unique_constraint("unique_filing_counter", ["filing", "counter"])
+
+    # run a counter of each submission in a given filing id, ordered by the submission id,
+    # which is the PK incrementor.  This will give us accurate counts of 1, 2, 3, ... for
+    # each submission per filing id.
+    conn = op.get_bind()
+    conn.execute(
+        sa.text(
+            """
+        WITH counts AS (
+            SELECT id, filing, ROW_NUMBER() OVER (PARTITION BY filing ORDER BY id) AS row_num FROM submission
+        )
+        UPDATE submission SET counter = counts.row_num
+        FROM counts
+        WHERE submission.id = counts.id
+        """
+        )
+    )
+
+    # set the counter column to required now that existing data is set.
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.alter_column("counter", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_constraint(constraint_name="unique_filing_counter", table_name="submission")
+    op.drop_column("submission", "counter")

src/sbl_filing_api/entities/models/dao.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import Any, List
 from sqlalchemy import Enum as SAEnum, String
-from sqlalchemy import ForeignKey, func
+from sqlalchemy import ForeignKey, func, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, relationship
 from sqlalchemy.ext.asyncio import AsyncAttrs
 from sqlalchemy.types import JSON
@@ -26,6 +26,7 @@ class SubmissionDAO(Base):
     __tablename__ = "submission"
     id: Mapped[int] = mapped_column(index=True, primary_key=True, autoincrement=True)
     filing: Mapped[int] = mapped_column(ForeignKey("filing.id"))
+    counter: Mapped[int]
     submitter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"))
     submitter: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[submitter_id])
     accepter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"), nullable=True)
@@ -37,6 +38,8 @@ class SubmissionDAO(Base):
     filename: Mapped[str]
     total_records: Mapped[int] = mapped_column(nullable=True)
 
+    __table_args__ = (UniqueConstraint("filing", "counter", name="unique_filing_counter"),)
+
     def __str__(self):
         return f"Submission ID: {self.id}, State: {self.state}, Ruleset: {self.validation_ruleset_version}, Filing Period: {self.filing}, Submission: {self.submission_time}"
 

src/sbl_filing_api/entities/models/dto.py

@@ -18,6 +18,7 @@ class SubmissionDTO(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     id: int | None = None
+    counter: int
     state: SubmissionState | None = None
     validation_ruleset_version: str | None = None
     validation_results: Dict[str, Any] | None = None

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -98,8 +98,15 @@ async def get_user_actions(session: AsyncSession) -> List[UserActionDAO]:
 
 
 async def add_submission(session: AsyncSession, filing_id: int, filename: str, submitter_id: int) -> SubmissionDAO:
+    stmt = select(SubmissionDAO).filter_by(filing=filing_id).order_by(desc(SubmissionDAO.counter)).limit(1)
+    last_sub = await session.scalar(stmt)
+    current_count = last_sub.counter if last_sub else 0
     new_sub = SubmissionDAO(
-        filing=filing_id, state=SubmissionState.SUBMISSION_STARTED, filename=filename, submitter_id=submitter_id
+        filing=filing_id,
+        state=SubmissionState.SUBMISSION_STARTED,
+        filename=filename,
+        submitter_id=submitter_id,
+        counter=(current_count + 1),
     )
     # this returns the attached object, most importantly with the new submission id
     new_sub = await session.merge(new_sub)

src/sbl_filing_api/routers/filing.py

@@ -151,7 +151,9 @@ async def sign_filing(request: Request, lei: str, period_code: str):
             action_type=UserActionType.SIGN,
         ),
     )
-    filing.confirmation_id = lei + "-" + period_code + "-" + str(latest_sub.id) + "-" + str(sig.timestamp.timestamp())
+    filing.confirmation_id = (
+        lei + "-" + period_code + "-" + str(latest_sub.id) + "-" + str(int(sig.timestamp.timestamp()))
+    )
     filing.signatures.append(sig)
     return await repo.upsert_filing(request.state.db_session, filing)
 

tests/api/routers/test_filing_api.py

@@ -160,6 +160,7 @@ async def test_get_submissions(self, mocker: MockerFixture, app_fixture: FastAPI
         mock.return_value = [
             SubmissionDAO(
                 filing=1,
+                counter=1,
                 state=SubmissionState.SUBMISSION_UPLOADED,
                 validation_ruleset_version="v1",
                 submission_time=datetime.datetime.now(),
@@ -208,6 +209,7 @@ async def test_get_latest_submission(
         mock = mocker.patch("sbl_filing_api.entities.repos.submission_repo.get_latest_submission")
         mock.return_value = SubmissionDAO(
             filing=1,
+            counter=1,
             state=SubmissionState.VALIDATION_IN_PROGRESS,
             validation_ruleset_version="v1",
             submission_time=datetime.datetime.now(),
@@ -254,6 +256,7 @@ async def test_get_submission_by_id(self, mocker: MockerFixture, app_fixture: Fa
         mock.return_value = SubmissionDAO(
             id=1,
             filing=1,
+            counter=1,
             state=SubmissionState.VALIDATION_WITH_ERRORS,
             validation_ruleset_version="v1",
             submission_time=datetime.datetime.now(),
@@ -293,6 +296,7 @@ def test_authed_upload_file(
         return_sub = SubmissionDAO(
             id=1,
             filing=1,
+            counter=1,
             state=SubmissionState.SUBMISSION_UPLOADED,
             filename="submission.csv",
             submitter_id=1,
@@ -384,6 +388,7 @@ def test_submission_update_fail(
         return_sub = SubmissionDAO(
             id=1,
             filing=1,
+            counter=1,
             state=SubmissionState.SUBMISSION_UPLOADED,
             filename="submission.csv",
         )
@@ -440,6 +445,7 @@ def test_submission_second_update_fail(
         return_sub = SubmissionDAO(
             id=1,
             filing=1,
+            counter=1,
             state=SubmissionState.SUBMISSION_UPLOADED,
             filename="submission.csv",
         )
@@ -915,6 +921,7 @@ async def test_accept_submission(self, mocker: MockerFixture, app_fixture: FastA
         mock.return_value = SubmissionDAO(
             id=1,
             filing=1,
+            counter=1,
             state=SubmissionState.VALIDATION_WITH_ERRORS,
             validation_ruleset_version="v1",
             submission_time=datetime.datetime.now(),
@@ -930,6 +937,7 @@ async def test_accept_submission(self, mocker: MockerFixture, app_fixture: FastA
         update_mock.return_value = SubmissionDAO(
             id=1,
             filing=1,
+            counter=2,
             state=SubmissionState.SUBMISSION_ACCEPTED,
             validation_ruleset_version="v1",
             submission_time=datetime.datetime.now(),
@@ -1024,7 +1032,7 @@ async def test_good_sign_filing(
         assert upsert_mock.call_args.args[1].confirmation_id.startswith("1234567890ABCDEFGH00-2024-1-")
         assert res.status_code == 200
         assert float(upsert_mock.call_args.args[1].confirmation_id.split("-")[3]) == pytest.approx(
-            dt.now().timestamp(), abs=1.5
+            int(dt.now().timestamp()), abs=1.5
         )
 
     async def test_errors_sign_filing(

tests/entities/repos/test_submission_repo.py

@@ -140,6 +140,7 @@ async def setup(
             validation_ruleset_version="v1",
             submission_time=dt.now(),
             filename="file1.csv",
+            counter=1,
         )
 
         submission2 = SubmissionDAO(
@@ -150,6 +151,7 @@ async def setup(
             validation_ruleset_version="v1",
             submission_time=(dt.now() - datetime.timedelta(seconds=200)),
             filename="file2.csv",
+            counter=1,
         )
         submission3 = SubmissionDAO(
             id=3,
@@ -159,6 +161,7 @@ async def setup(
             validation_ruleset_version="v1",
             submission_time=dt.now(),
             filename="file3.csv",
+            counter=2,
         )
         submission4 = SubmissionDAO(
             id=4,
@@ -167,6 +170,7 @@ async def setup(
             validation_ruleset_version="v1",
             submission_time=(dt.now() - datetime.timedelta(seconds=400)),
             filename="file4.csv",
+            counter=2,
         )
         submission1.submitter = user_action2
         submission2.submitter = user_action2
@@ -464,6 +468,7 @@ async def test_add_submission(self, transaction_session: AsyncSession):
         )
         assert res.id == 5
         assert res.filing == 1
+        assert res.counter == 3
         assert res.state == SubmissionState.SUBMISSION_STARTED
         assert res.submitter.id == user_action_submit.id
         assert res.submitter.user_id == user_action_submit.user_id
@@ -518,6 +523,7 @@ async def query_updated_dao():
                 new_res2 = await search_session.scalar(stmt)
                 assert new_res2.id == 5
                 assert new_res2.filing == 1
+                assert new_res2.counter == 3
                 assert new_res2.state == SubmissionState.VALIDATION_WITH_ERRORS
                 assert new_res2.validation_results == validation_results
 

tests/migrations/test_migrations.py

@@ -442,3 +442,16 @@ def test_migrations_to_26f11ac15b3c(alembic_runner: MigrationContext, alembic_en
     inspector = sqlalchemy.inspect(alembic_engine)
 
     assert "is_voluntary" in set([c["name"] for c in inspector.get_columns("filing")])
+
+
+def test_migrations_to_6ec12afa5b37(alembic_runner: MigrationContext, alembic_engine: Engine):
+    alembic_runner.migrate_up_to("6ec12afa5b37")
+
+    inspector = sqlalchemy.inspect(alembic_engine)
+
+    counter_constraint = inspector.get_unique_constraints("submission")[0]
+
+    assert "counter" in set([c["name"] for c in inspector.get_columns("submission")])
+    assert counter_constraint["name"] == "unique_filing_counter"
+    print(f"{counter_constraint["column_names"]}")
+    assert set(counter_constraint["column_names"]) == set(["filing", "counter"])