Skip to content

Commit

Permalink
Apparent success on local DB
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonaraphael
jonaraphael committed Aug 22, 2024
1 parent cc07642 commit 3438171
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 123 deletions.
74 changes: 15 additions & 59 deletions alembic/versions/3c4693517ef6_add_tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,13 +157,10 @@ def upgrade() -> None:
)

op.create_table(
"users",
"user",
sa.Column("id", sa.BigInteger, primary_key=True),
sa.Column("name", sa.Text),
sa.Column("email", sa.Text),
sa.Column("emailVerified", sa.DateTime),
sa.Column("image", sa.Text),
sa.Column("role", sa.Text),
sa.Column("email", sa.Text, nullable=False, unique=True),
sa.Column("create_time", sa.DateTime, server_default=sa.func.now()),
)

op.create_table(
Expand All @@ -183,7 +180,7 @@ def upgrade() -> None:
op.create_table(
"subscription",
sa.Column("id", sa.BigInteger, primary_key=True),
sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False),
sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id"), nullable=False),
sa.Column("filter", sa.BigInteger, sa.ForeignKey("filter.id"), nullable=False),
sa.Column(
"frequency", sa.Integer, sa.ForeignKey("frequency.id"), nullable=False
Expand All @@ -194,35 +191,14 @@ def upgrade() -> None:
)

op.create_table(
"verification_token",
sa.Column("identifier", sa.Text, nullable=False),
sa.Column("expires", sa.DateTime, nullable=False),
sa.Column("token", sa.Text, nullable=False),
sa.PrimaryKeyConstraint("identifier", "token"),
)

op.create_table(
"accounts",
sa.Column("id", sa.BigInteger, primary_key=True),
sa.Column("userId", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False),
sa.Column("type", sa.Text, nullable=False),
sa.Column("provider", sa.Text, nullable=False),
sa.Column("providerAccountId", sa.Text, nullable=False),
sa.Column("refresh_token", sa.Text),
sa.Column("access_token", sa.Text),
sa.Column("expires_at", sa.BigInteger),
sa.Column("id_token", sa.Text),
sa.Column("scope", sa.Text),
sa.Column("session_state", sa.Text),
sa.Column("token_type", sa.Text),
)

op.create_table(
"sessions",
"magic_link",
sa.Column("id", sa.BigInteger, primary_key=True),
sa.Column("userId", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False),
sa.Column("expires", sa.DateTime, nullable=False),
sa.Column("sessionToken", sa.Text, nullable=False),
sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id"), nullable=False),
sa.Column("token", sa.Text, nullable=False),
sa.Column("expiration_time", sa.DateTime, nullable=False),
sa.Column("is_used", sa.Boolean, nullable=False),
sa.Column("create_time", sa.DateTime, server_default=sa.func.now()),
sa.Column("update_time", sa.DateTime, server_default=sa.func.now()),
)

op.create_table(
Expand Down Expand Up @@ -271,7 +247,7 @@ def upgrade() -> None:
op.create_table(
"aoi_user",
sa.Column("aoi_id", sa.BigInteger, sa.ForeignKey("aoi.id"), primary_key=True),
sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id")),
sa.Column("user", sa.BigInteger, sa.ForeignKey("user.id")),
sa.Column("create_time", sa.DateTime, server_default=sa.func.now()),
)

Expand Down Expand Up @@ -356,54 +332,34 @@ def upgrade() -> None:
sa.Column("source", sa.BigInteger, sa.ForeignKey("source.id"), nullable=False),
sa.Column("coincidence_score", sa.Float),
sa.Column("rank", sa.BigInteger),
sa.Column("hitl_confirmed", sa.Boolean),
sa.Column("geojson_fc", sa.JSON, nullable=False),
sa.Column("geometry", Geography("GEOMETRY"), nullable=False),
sa.Column(
"create_time", sa.DateTime, nullable=False, server_default=sa.func.now()
),
sa.Column("hitl_verification", sa.Boolean),
sa.Column("hitl_confidence", sa.Float),
sa.Column("hitl_user", sa.BigInteger, sa.ForeignKey("users.id")),
sa.Column("hitl_time", sa.DateTime),
sa.Column("hitl_notes", sa.Text),
)

op.create_table(
"hitl_slick",
sa.Column("id", sa.BigInteger, primary_key=True),
sa.Column("slick", sa.BigInteger, sa.ForeignKey("slick.id"), nullable=False),
sa.Column("user", sa.BigInteger, sa.ForeignKey("users.id"), nullable=False),
sa.Column("cls", sa.BigInteger, sa.ForeignKey("cls.id"), nullable=False),
sa.Column("confidence", sa.Float),
sa.Column(
"update_time", sa.DateTime, nullable=False, server_default=sa.func.now()
),
)


def downgrade() -> None:
"""drop tables"""
op.drop_table("hitl_slick")
op.drop_table("slick_to_source")
op.drop_table("source_infra")
op.drop_table("source_vessel")
op.drop_table("source")
op.drop_table("source_type")
op.drop_table("slick_to_aoi")
op.drop_table("aoi_chunks")
op.drop_table("aoi_user")
op.drop_table("aoi_mpa")
op.drop_table("aoi_iho")
op.drop_table("aoi_eez")
op.drop_table("aoi")
op.drop_table("aoi_type")
op.drop_table("verification_token")
op.drop_table("sessions")
op.drop_table("accounts")
op.drop_table("magic_link")
op.drop_table("subscription")
op.drop_table("frequency")
op.drop_table("filter")
op.drop_table("users")
op.drop_table("user")
op.drop_table("slick")
op.drop_table("cls")
op.drop_table("orchestrator_run")
Expand Down
19 changes: 10 additions & 9 deletions alembic/versions/96cd87e159a9_prepare_db_for_cv3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,23 @@ def upgrade() -> None:
"""

# Drop the old foreign key constraints
op.drop_constraint("fk_subscription_user_id", "subscription", type_="foreignkey")
op.drop_constraint("fk_aoi_user_user_id", "aoi_user", type_="foreignkey")
op.drop_constraint("subscription_user_fkey", "subscription", type_="foreignkey")
op.drop_constraint("aoi_user_user_fkey", "aoi_user", type_="foreignkey")
op.drop_constraint("magic_link_user_fkey", "magic_link", type_="foreignkey")

# Alter user table and create new structure as 'users'
op.rename_table("user", "users")
op.add_column("users", sa.Column("name", sa.Text))
op.add_column("users", sa.Column("emailVerified", sa.DateTime))
op.add_column("users", sa.Column("image", sa.Text))
op.add_column("users", sa.Column("role", sa.Text))
op.alter_column(
"users", "email", new_column_name="email", type_=sa.Text, nullable=True
)
op.create_foreign_key(None, "subscription", "users", ["user"], ["id"])
op.alter_column("users", "email", nullable=True)
op.drop_column("users", "create_time")

op.create_foreign_key(None, "subscription", "users", ["user"], ["id"])
op.create_foreign_key(None, "aoi_user", "users", ["user"], ["id"])
op.create_foreign_key(None, "magic_link", "users", ["user"], ["id"])

# Remove hitl_confirmed column
op.drop_column("slick_to_source", "hitl_confirmed")

Expand Down Expand Up @@ -85,6 +87,7 @@ def upgrade() -> None:
op.add_column(
"verification_token", sa.Column("expires", sa.DateTime, nullable=False)
)
op.drop_constraint("magic_link_pkey", "verification_token", type_="primary")
op.create_primary_key(
"verification_token_pk", "verification_token", ["identifier", "token"]
)
Expand Down Expand Up @@ -248,9 +251,7 @@ def downgrade() -> None:
op.drop_column("users", "emailVerified")
op.drop_column("users", "image")
op.drop_column("users", "role")
op.alter_column(
"users", "email", new_column_name="email", type_=sa.String(255), nullable=False
)
op.alter_column("users", "email", nullable=False)
op.rename_table("users", "user")

# Recreate foreign key constraints
Expand Down
60 changes: 5 additions & 55 deletions alembic/versions/c941681a050d_add_initial_records.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,14 @@ def upgrade() -> None:
long_name="Vessel, coincident",
supercls=7,
),
database_schema.Cls(
short_name="AMBIGUOUS",
long_name="Ambiguous",
),
]
session.add_all(clses)

models = [
database_schema.Model(
type="MASKRCNN",
file_path="experiments/2023_10_05_02_22_46_4cls_rnxt101_pr512_px1024_680min_maskrcnn_wd01/scripting_cpu_model.pt",
layers=["VV", "ALL_255", "VESSEL"],
layers=["VV", "INFRA", "VESSEL"],
cls_map={
0: "BACKGROUND",
1: "INFRA",
Expand All @@ -88,43 +84,16 @@ def upgrade() -> None:
tile_width_px=512,
epochs=122,
thresholds={
"poly_nms_thresh": 0.2,
"pixel_nms_thresh": 0.4,
"bbox_score_thresh": 0.3,
"poly_score_thresh": 0.1,
"pixel_score_thresh": 0.5,
"bbox_score_thresh": 0.2,
"poly_score_thresh": 0.2,
"pixel_score_thresh": 0.2,
"groundtruth_dice_thresh": 0.0,
},
backbone_size=101,
pixel_f1=0.461,
instance_f1=0.47,
),
database_schema.Model(
type="FASTAIUNET",
file_path="experiments/2024_08_18_06_27_25_4cls_resnet34_pr512_px1024_500epochs_unet/tracing_cpu_model.pt",
layers=["VV"],
cls_map={
0: "BACKGROUND",
1: "INFRA",
2: "NATURAL",
3: "VESSEL",
}, # inference_idx maps to class table
name="ResNet34 41%",
tile_width_m=40844, # Used to calculate zoom
tile_width_px=512, # Used to calculate scale
epochs=500,
thresholds={
"poly_nms_thresh": 0.2,
"pixel_nms_thresh": 0.0, # NOT USED IN UNETS
"bbox_score_thresh": 0.01,
"poly_score_thresh": 0.3,
"pixel_score_thresh": 0.8,
"groundtruth_dice_thresh": 0.0,
},
backbone_size=34,
pixel_f1=0.536,
# instance_f1=0.0, # TODO CALCULATE
),
]
session.add_all(models)

Expand All @@ -148,20 +117,6 @@ def upgrade() -> None:
source_url="https://gmtds.maplarge.com/public/ext/GMTDS/Main",
notes="Typically uses the previous month's density map. If unavailable will default to previous year.",
),
database_schema.Layer(
short_name="ALL_255", # TODO Rename to something liketo avoid conflict with PIXEL CLASS
long_name="All Pixels Value=255",
citation="",
source_url="",
notes="Can be used for ablation or to replace unwanted layers.",
),
database_schema.Layer(
short_name="ALL_ZEROS", # TODO Rename to something liketo avoid conflict with PIXEL CLASS
long_name="All Pixels Value=0",
citation="",
source_url="",
notes="Can be used for ablation or to replace unwanted layers.",
),
]
session.add_all(layers)

Expand Down Expand Up @@ -248,11 +203,7 @@ def downgrade() -> None:

layers = (
session.query(database_schema.Layer)
.filter(
database_schema.Layer.short_name.in_(
["VV", "INFRA", "VESSEL", "ALL_255", "ALL_ZEROS"]
)
)
.filter(database_schema.Layer.short_name.in_(["VV", "INFRA", "VESSEL"]))
.all()
)
for layer in layers:
Expand Down Expand Up @@ -301,7 +252,6 @@ def downgrade() -> None:
"OLD_VESSEL",
"REC_VESSEL",
"COIN_VESSEL",
"AMBIGUOUS",
]
)
)
Expand Down

0 comments on commit 3438171

Please sign in to comment.