diff --git a/alembic/versions/3c4693517ef6_add_tables.py b/alembic/versions/3c4693517ef6_add_tables.py index 84687a65..e555dcbd 100644 --- a/alembic/versions/3c4693517ef6_add_tables.py +++ b/alembic/versions/3c4693517ef6_add_tables.py @@ -157,13 +157,10 @@ def upgrade() -> None: ) op.create_table( - "users", + "user", sa.Column("id", sa.BigInteger, primary_key=True), - sa.Column("name", sa.Text), - sa.Column("email", sa.Text), - sa.Column("emailVerified", sa.DateTime), - sa.Column("image", sa.Text), - sa.Column("role", sa.Text), + sa.Column("email", sa.Text, nullable=False, unique=True), + sa.Column("create_time", sa.DateTime, server_default=sa.func.now()), ) op.create_table( @@ -183,7 +180,7 @@ def upgrade() -> None: op.create_table( "subscription", sa.Column("id", sa.BigInteger, primary_key=True), - sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False), + sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id"), nullable=False), sa.Column("filter", sa.BigInteger, sa.ForeignKey("filter.id"), nullable=False), sa.Column( "frequency", sa.Integer, sa.ForeignKey("frequency.id"), nullable=False @@ -194,35 +191,14 @@ def upgrade() -> None: ) op.create_table( - "verification_token", - sa.Column("identifier", sa.Text, nullable=False), - sa.Column("expires", sa.DateTime, nullable=False), - sa.Column("token", sa.Text, nullable=False), - sa.PrimaryKeyConstraint("identifier", "token"), - ) - - op.create_table( - "accounts", - sa.Column("id", sa.BigInteger, primary_key=True), - sa.Column("userId", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False), - sa.Column("type", sa.Text, nullable=False), - sa.Column("provider", sa.Text, nullable=False), - sa.Column("providerAccountId", sa.Text, nullable=False), - sa.Column("refresh_token", sa.Text), - sa.Column("access_token", sa.Text), - sa.Column("expires_at", sa.BigInteger), - sa.Column("id_token", sa.Text), - sa.Column("scope", sa.Text), - sa.Column("session_state", sa.Text), - sa.Column("token_type", sa.Text), - ) - - op.create_table( - "sessions", + "magic_link", sa.Column("id", sa.BigInteger, primary_key=True), - sa.Column("userId", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False), - sa.Column("expires", sa.DateTime, nullable=False), - sa.Column("sessionToken", sa.Text, nullable=False), + sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id"), nullable=False), + sa.Column("token", sa.Text, nullable=False), + sa.Column("expiration_time", sa.DateTime, nullable=False), + sa.Column("is_used", sa.Boolean, nullable=False), + sa.Column("create_time", sa.DateTime, server_default=sa.func.now()), + sa.Column("update_time", sa.DateTime, server_default=sa.func.now()), ) op.create_table( @@ -271,7 +247,7 @@ def upgrade() -> None: op.create_table( "aoi_user", sa.Column("aoi_id", sa.BigInteger, sa.ForeignKey("aoi.id"), primary_key=True), - sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id")), + sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id")), sa.Column("create_time", sa.DateTime, server_default=sa.func.now()), ) @@ -356,54 +332,34 @@ def upgrade() -> None: sa.Column("source", sa.BigInteger, sa.ForeignKey("source.id"), nullable=False), sa.Column("coincidence_score", sa.Float), sa.Column("rank", sa.BigInteger), + sa.Column("hitl_confirmed", sa.Boolean), sa.Column("geojson_fc", sa.JSON, nullable=False), sa.Column("geometry", Geography("GEOMETRY"), nullable=False), sa.Column( "create_time", sa.DateTime, nullable=False, server_default=sa.func.now() ), - sa.Column("hitl_verification", sa.Boolean), - sa.Column("hitl_confidence", sa.Float), - sa.Column("hitl_user", sa.BigInteger, sa.ForeignKey("users.id")), - sa.Column("hitl_time", sa.DateTime), - sa.Column("hitl_notes", sa.Text), - ) - - op.create_table( - "hitl_slick", - sa.Column("id", sa.BigInteger, primary_key=True), - sa.Column("slick", sa.BigInteger, sa.ForeignKey("slick.id"), nullable=False), - sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False), - sa.Column("cls", sa.BigInteger, sa.ForeignKey("cls.id"), nullable=False), - sa.Column("confidence", sa.Float), - sa.Column( - "update_time", sa.DateTime, nullable=False, server_default=sa.func.now() - ), ) def downgrade() -> None: """drop tables""" - op.drop_table("hitl_slick") op.drop_table("slick_to_source") op.drop_table("source_infra") op.drop_table("source_vessel") op.drop_table("source") op.drop_table("source_type") op.drop_table("slick_to_aoi") - op.drop_table("aoi_chunks") op.drop_table("aoi_user") op.drop_table("aoi_mpa") op.drop_table("aoi_iho") op.drop_table("aoi_eez") op.drop_table("aoi") op.drop_table("aoi_type") - op.drop_table("verification_token") - op.drop_table("sessions") - op.drop_table("accounts") + op.drop_table("magic_link") op.drop_table("subscription") op.drop_table("frequency") op.drop_table("filter") - op.drop_table("users") + op.drop_table("user") op.drop_table("slick") op.drop_table("cls") op.drop_table("orchestrator_run") diff --git a/alembic/versions/96cd87e159a9_prepare_db_for_cv3.py b/alembic/versions/96cd87e159a9_prepare_db_for_cv3.py index 67abeba2..3bce6ed0 100644 --- a/alembic/versions/96cd87e159a9_prepare_db_for_cv3.py +++ b/alembic/versions/96cd87e159a9_prepare_db_for_cv3.py @@ -37,8 +37,9 @@ def upgrade() -> None: """ # Drop the old foreign key constraints - op.drop_constraint("fk_subscription_user_id", "subscription", type_="foreignkey") - op.drop_constraint("fk_aoi_user_user_id", "aoi_user", type_="foreignkey") + op.drop_constraint("subscription_user_fkey", "subscription", type_="foreignkey") + op.drop_constraint("aoi_user_user_fkey", "aoi_user", type_="foreignkey") + op.drop_constraint("magic_link_user_fkey", "magic_link", type_="foreignkey") # Alter user table and create new structure as 'users' op.rename_table("user", "users") @@ -46,12 +47,13 @@ def upgrade() -> None: op.add_column("users", sa.Column("emailVerified", sa.DateTime)) op.add_column("users", sa.Column("image", sa.Text)) op.add_column("users", sa.Column("role", sa.Text)) - op.alter_column( - "users", "email", new_column_name="email", type_=sa.Text, nullable=True - ) - op.create_foreign_key(None, "subscription", "users", ["user"], ["id"]) + op.alter_column("users", "email", nullable=True) op.drop_column("users", "create_time") + op.create_foreign_key(None, "subscription", "users", ["user"], ["id"]) + op.create_foreign_key(None, "aoi_user", "users", ["user"], ["id"]) + op.create_foreign_key(None, "magic_link", "users", ["user"], ["id"]) + # Remove hitl_confirmed column op.drop_column("slick_to_source", "hitl_confirmed") @@ -85,6 +87,7 @@ def upgrade() -> None: op.add_column( "verification_token", sa.Column("expires", sa.DateTime, nullable=False) ) + op.drop_constraint("magic_link_pkey", "verification_token", type_="primary") op.create_primary_key( "verification_token_pk", "verification_token", ["identifier", "token"] ) @@ -248,9 +251,7 @@ def downgrade() -> None: op.drop_column("users", "emailVerified") op.drop_column("users", "image") op.drop_column("users", "role") - op.alter_column( - "users", "email", new_column_name="email", type_=sa.String(255), nullable=False - ) + op.alter_column("users", "email", nullable=False) op.rename_table("users", "user") # Recreate foreign key constraints diff --git a/alembic/versions/c941681a050d_add_initial_records.py b/alembic/versions/c941681a050d_add_initial_records.py index 00f0ddd5..01da4418 100644 --- a/alembic/versions/c941681a050d_add_initial_records.py +++ b/alembic/versions/c941681a050d_add_initial_records.py @@ -65,10 +65,6 @@ def upgrade() -> None: long_name="Vessel, coincident", supercls=7, ), - database_schema.Cls( - short_name="AMBIGUOUS", - long_name="Ambiguous", - ), ] session.add_all(clses) @@ -76,7 +72,7 @@ def upgrade() -> None: database_schema.Model( type="MASKRCNN", file_path="experiments/2023_10_05_02_22_46_4cls_rnxt101_pr512_px1024_680min_maskrcnn_wd01/scripting_cpu_model.pt", - layers=["VV", "ALL_255", "VESSEL"], + layers=["VV", "INFRA", "VESSEL"], cls_map={ 0: "BACKGROUND", 1: "INFRA", @@ -88,43 +84,16 @@ def upgrade() -> None: tile_width_px=512, epochs=122, thresholds={ - "poly_nms_thresh": 0.2, "pixel_nms_thresh": 0.4, - "bbox_score_thresh": 0.3, - "poly_score_thresh": 0.1, - "pixel_score_thresh": 0.5, + "bbox_score_thresh": 0.2, + "poly_score_thresh": 0.2, + "pixel_score_thresh": 0.2, "groundtruth_dice_thresh": 0.0, }, backbone_size=101, pixel_f1=0.461, instance_f1=0.47, ), - database_schema.Model( - type="FASTAIUNET", - file_path="experiments/2024_08_18_06_27_25_4cls_resnet34_pr512_px1024_500epochs_unet/tracing_cpu_model.pt", - layers=["VV"], - cls_map={ - 0: "BACKGROUND", - 1: "INFRA", - 2: "NATURAL", - 3: "VESSEL", - }, # inference_idx maps to class table - name="ResNet34 41%", - tile_width_m=40844, # Used to calculate zoom - tile_width_px=512, # Used to calculate scale - epochs=500, - thresholds={ - "poly_nms_thresh": 0.2, - "pixel_nms_thresh": 0.0, # NOT USED IN UNETS - "bbox_score_thresh": 0.01, - "poly_score_thresh": 0.3, - "pixel_score_thresh": 0.8, - "groundtruth_dice_thresh": 0.0, - }, - backbone_size=34, - pixel_f1=0.536, - # instance_f1=0.0, # TODO CALCULATE - ), ] session.add_all(models) @@ -148,20 +117,6 @@ def upgrade() -> None: source_url="https://gmtds.maplarge.com/public/ext/GMTDS/Main", notes="Typically uses the previous month's density map. If unavailable will default to previous year.", ), - database_schema.Layer( - short_name="ALL_255", # TODO Rename to something liketo avoid conflict with PIXEL CLASS - long_name="All Pixels Value=255", - citation="", - source_url="", - notes="Can be used for ablation or to replace unwanted layers.", - ), - database_schema.Layer( - short_name="ALL_ZEROS", # TODO Rename to something liketo avoid conflict with PIXEL CLASS - long_name="All Pixels Value=0", - citation="", - source_url="", - notes="Can be used for ablation or to replace unwanted layers.", - ), ] session.add_all(layers) @@ -248,11 +203,7 @@ def downgrade() -> None: layers = ( session.query(database_schema.Layer) - .filter( - database_schema.Layer.short_name.in_( - ["VV", "INFRA", "VESSEL", "ALL_255", "ALL_ZEROS"] - ) - ) + .filter(database_schema.Layer.short_name.in_(["VV", "INFRA", "VESSEL"])) .all() ) for layer in layers: @@ -301,7 +252,6 @@ def downgrade() -> None: "OLD_VESSEL", "REC_VESSEL", "COIN_VESSEL", - "AMBIGUOUS", ] ) )