Add partitioned submission table

.pylintrc

@@ -422,4 +422,4 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

onadata/apps/logger/management/commands/import_instances.py

@@ -26,12 +26,15 @@ class Command(BaseCommand):
     import_instances - import ODK instances from a zipped file.
     """
 
-    args = "username path"
     help = gettext_lazy(
         "Import a zip file, a directory containing zip files "
         "or a directory of ODK instances"
     )
 
+    def add_arguments(self, parser):
+        parser.add_argument("username", type=str)
+        parser.add_argument("path", type=str)
+
     def _log_import(self, results):
         total_count, success_count, errors = results
         self.stdout.write(
@@ -44,10 +47,8 @@ def _log_import(self, results):
 
     # pylint: disable=unused-argument
     def handle(self, *args, **kwargs):
-        if len(args) < 2:
-            raise CommandError(_("Usage: <command> username file/path."))
-        username = args[0]
-        path = args[1]
+        username = kwargs["username"]
+        path = kwargs["path"]
         is_async = False
         if len(args) > 2:
             if isinstance(args[2], str):
@@ -63,7 +64,6 @@ def handle(self, *args, **kwargs):
         # make sure path exists
         if not os.path.exists(path):
             raise CommandError(_(f"The specified path '{path}' does not exist."))
-
         for directory, subdirs, files in os.walk(path):
             # check if the directory has an odk directory
             if "odk" in subdirs:

onadata/apps/logger/migrations/0011_auto_20231120_0939.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.20 on 2023-11-20 14:39
+
+from django.db import migrations, models
+import onadata.apps.logger.models.instance
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("logger", "0010_auto_20230921_0346"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="instance",
+            name="logger_inst_deleted_at_da31a3_idx",
+        ),
+        migrations.AlterField(
+            model_name="instance",
+            name="date_created",
+            field=models.DateTimeField(
+                blank=True,
+                default=onadata.apps.logger.models.instance.now,
+                editable=False,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="instance",
+            name="date_modified",
+            field=models.DateTimeField(
+                blank=True,
+                default=onadata.apps.logger.models.instance.now,
+                editable=False,
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="instance",
+            index=models.Index(
+                fields=["deleted_at"], name="logger_inst_deleted_da31a3_idx"
+            ),
+        ),
+    ]

onadata/apps/logger/migrations/0012_auto_20231122_0403.py

@@ -0,0 +1,195 @@
+# Generated by Django 3.2.20 on 2023-11-22 09:03
+
+from django.conf import settings
+import django.contrib.gis.db.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import psqlextra.backend.migrations.operations.add_default_partition
+import psqlextra.backend.migrations.operations.create_partitioned_model
+import psqlextra.manager.manager
+import psqlextra.models.partitioned
+import psqlextra.types
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("taggit", "0005_auto_20220424_2025"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("logger", "0011_auto_20231120_0939"),
+    ]
+
+    operations = [
+        psqlextra.backend.migrations.operations.create_partitioned_model.PostgresCreatePartitionedModel(
+            name="Submission",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("json", models.JSONField(default=dict)),
+                ("xml", models.TextField()),
+                (
+                    "date_created",
+                    models.DateTimeField(
+                        blank=True, default=django.utils.timezone.now, editable=False
+                    ),
+                ),
+                (
+                    "date_modified",
+                    models.DateTimeField(
+                        blank=True, default=django.utils.timezone.now, editable=False
+                    ),
+                ),
+                ("deleted_at", models.DateTimeField(default=None, null=True)),
+                ("last_edited", models.DateTimeField(default=None, null=True)),
+                (
+                    "status",
+                    models.CharField(default="submitted_via_web", max_length=20),
+                ),
+                ("uuid", models.CharField(db_index=True, default="", max_length=249)),
+                ("version", models.CharField(max_length=255, null=True)),
+                (
+                    "geom",
+                    django.contrib.gis.db.models.fields.GeometryCollectionField(
+                        null=True, srid=4326
+                    ),
+                ),
+                (
+                    "media_all_received",
+                    models.BooleanField(
+                        default=True,
+                        null=True,
+                        verbose_name="Received All Media Attachemts",
+                    ),
+                ),
+                (
+                    "total_media",
+                    models.PositiveIntegerField(
+                        default=0, null=True, verbose_name="Total Media Attachments"
+                    ),
+                ),
+                (
+                    "media_count",
+                    models.PositiveIntegerField(
+                        default=0, null=True, verbose_name="Received Media Attachments"
+                    ),
+                ),
+                (
+                    "checksum",
+                    models.CharField(
+                        blank=True, db_index=True, max_length=64, null=True
+                    ),
+                ),
+                (
+                    "has_a_review",
+                    models.BooleanField(default=False, verbose_name="has_a_review"),
+                ),
+                (
+                    "deleted_by",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="deleted_submissions",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "instance_id",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="submission",
+                        to="logger.instance",
+                    ),
+                ),
+                (
+                    "organization",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="org_submissions",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="logger.project"
+                    ),
+                ),
+                (
+                    "submitted_by",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="submissions",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "survey_type",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="logger.surveytype",
+                    ),
+                ),
+                (
+                    "tags",
+                    taggit.managers.TaggableManager(
+                        help_text="A comma-separated list of tags.",
+                        through="taggit.TaggedItem",
+                        to="taggit.Tag",
+                        verbose_name="Tags",
+                    ),
+                ),
+                (
+                    "xform",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="submissions",
+                        to="logger.xform",
+                    ),
+                ),
+            ],
+            partitioning_options={
+                "method": psqlextra.types.PostgresPartitioningMethod["LIST"],
+                "key": ["organization_id"],
+            },
+            bases=(psqlextra.models.partitioned.PostgresPartitionedModel,),
+            managers=[
+                ("objects", psqlextra.manager.manager.PostgresManager()),
+            ],
+        ),
+        psqlextra.backend.migrations.operations.add_default_partition.PostgresAddDefaultPartition(
+            model_name="Submission",
+            name="default",
+        ),
+        migrations.AddIndex(
+            model_name="submission",
+            index=models.Index(
+                fields=["date_created"], name="logger_subm_date_cr_5e4d71_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="submission",
+            index=models.Index(
+                fields=["date_modified"], name="logger_subm_date_mo_cacc29_idx"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="submission",
+            index=models.Index(
+                fields=["deleted_at"], name="logger_subm_deleted_1888ec_idx"
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="submission",
+            unique_together={("organization", "xform", "uuid")},
+        ),
+    ]

onadata/apps/logger/migrations/0013_merge_20240104_1328.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.20 on 2024-01-04 18:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("logger", "0011_add_xform_id_instance_id_idx"),
+        ("logger", "0012_auto_20231122_0403"),
+    ]
+
+    operations = []

onadata/apps/logger/migrations/0014_merge_20240318_0420.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.20 on 2024-03-18 08:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("logger", "0012_add_instance_history_uuid_and_checksum_idx"),
+        ("logger", "0013_merge_20240104_1328"),
+    ]
+
+    operations = []

onadata/apps/logger/models/__init__.py

@@ -17,3 +17,4 @@
 from onadata.apps.logger.models.xform_version import XFormVersion  # noqa
 from onadata.apps.logger.xform_instance_parser import InstanceParseError  # noqa
 from onadata.apps.logger.models.project_invitation import ProjectInvitation  # noqa
+from onadata.apps.logger.models.submission import Submission  # noqa