diff --git a/src/backend/app/db/enums.py b/src/backend/app/db/enums.py index 974f2d307..b8692b62e 100644 --- a/src/backend/app/db/enums.py +++ b/src/backend/app/db/enums.py @@ -234,6 +234,14 @@ class GeometryType(StrEnum, Enum): Point = "Point" +class DbGeomType(StrEnum, Enum): + """Enum in the database, all geom types are in caps.""" + + POINT = "POINT" + POLYGON = "POLYGON" + LINESTRING = "LINESTRING" + + class XLSFormType(StrEnum, Enum): """XLSForm categories bundled by default. diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index fc73a55a9..fe3be857b 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -34,7 +34,7 @@ from psycopg import Connection from psycopg.errors import UniqueViolation from psycopg.rows import class_row -from pydantic import AwareDatetime, BaseModel, Field, ValidationInfo +from pydantic import AwareDatetime, BaseModel, Field, PositiveInt, ValidationInfo from pydantic.functional_validators import field_validator from app.central.central_schemas import ODKCentralDecrypted @@ -43,6 +43,7 @@ from app.db.enums import ( BackgroundTaskStatus, CommunityType, + DbGeomType, EntityState, GeomStatus, HTTPStatus, @@ -330,6 +331,7 @@ class DbOrganisation(BaseModel): type: Optional[OrganisationType] = None approved: Optional[bool] = None created_by: Optional[int] = None # this is not foreign key linked intentionally + associated_email: Optional[str] = None odk_central_url: Optional[str] = None odk_central_user: Optional[str] = None odk_central_password: Optional[str] = None @@ -967,6 +969,9 @@ class DbProject(BaseModel): data_extract_url: Optional[str] = None task_split_dimension: Optional[int] = None task_num_buildings: Optional[int] = None + new_geom_type: Optional[DbGeomType] = None + geo_restrict_distance_meters: Optional[PositiveInt] = None + geo_restrict_force_error: Optional[bool] = None hashtags: Optional[list[str]] = None due_date: Optional[AwareDatetime] = None updated_at: Optional[AwareDatetime] = None diff --git a/src/backend/app/organisations/organisation_crud.py b/src/backend/app/organisations/organisation_crud.py index 43c3a50ae..096095c8d 100644 --- a/src/backend/app/organisations/organisation_crud.py +++ b/src/backend/app/organisations/organisation_crud.py @@ -66,6 +66,7 @@ async def init_admin_org(db: Connection) -> None: name="HOTOSM", description="Humanitarian OpenStreetMap Team.", url="https://hotosm.org", + associated_email="sysadmin@hotosm.org", odk_central_url=settings.ODK_CENTRAL_URL, odk_central_user=settings.ODK_CENTRAL_USER, odk_central_password=settings.ODK_CENTRAL_PASSWD.get_secret_value() diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index b9a51c3aa..7939c6be1 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -107,4 +107,5 @@ class OrganisationOut(BaseModel): description: Optional[str] slug: Optional[str] url: Optional[str] + associated_email: Optional[str] odk_central_url: Optional[str] diff --git a/src/backend/migrations/002-create-geometry-log.sql b/src/backend/migrations/002-create-geometry-log.sql index 8a7a4615e..b3c7cd233 100644 --- a/src/backend/migrations/002-create-geometry-log.sql +++ b/src/backend/migrations/002-create-geometry-log.sql @@ -4,13 +4,14 @@ BEGIN; -CREATE TYPE public.geomstatus AS ENUM ( - 'BAD', - 'NEW' -); +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomstatus') THEN + CREATE TYPE public.geomstatus AS ENUM ('BAD', 'NEW'); + END IF; +END $$; ALTER TYPE public.geomstatus OWNER TO fmtm; -CREATE TABLE public.geometrylog ( +CREATE TABLE IF NOT EXISTS public.geometrylog ( id SERIAL PRIMARY KEY, geom GEOMETRY NOT NULL, status geomstatus, @@ -20,7 +21,12 @@ CREATE TABLE public.geometrylog ( ALTER TABLE public.geometrylog OWNER TO fmtm; -- Indexes for efficient querying -CREATE INDEX idx_geometrylog ON geometrylog USING gist (geom); +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN + CREATE INDEX IF NOT EXISTS idx_geometrylog ON geometrylog USING gist (geom); + END IF; +END $$; -- Commit the transaction COMMIT; diff --git a/src/backend/migrations/003-jsonb-geom-geometry-log.sql b/src/backend/migrations/003-jsonb-geom-geometry-log.sql index 5048d57c1..d1730c6ac 100644 --- a/src/backend/migrations/003-jsonb-geom-geometry-log.sql +++ b/src/backend/migrations/003-jsonb-geom-geometry-log.sql @@ -3,12 +3,25 @@ -- Start a transaction BEGIN; + -- drop existing indexes DROP INDEX IF EXISTS idx_geometrylog; -- Change the 'geom' column to jsonb type -ALTER TABLE geometrylog -ALTER COLUMN geom TYPE jsonb USING geom::jsonb; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN + ALTER TABLE geometrylog + ALTER COLUMN geom TYPE jsonb USING geom::jsonb; + END IF; +END $$; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geojson') THEN + ALTER TABLE geometrylog + ALTER COLUMN geojson TYPE JSONB USING geojson::jsonb; + END IF; +END $$; -- Alter the 'id' column to UUID -- set the default value using gen_random_uuid() @@ -21,7 +34,12 @@ ALTER TABLE geometrylog ALTER COLUMN id TYPE UUID USING gen_random_uuid(), ALTER COLUMN id SET DEFAULT gen_random_uuid(); -CREATE INDEX IF NOT EXISTS idx_geom_gin ON geometrylog USING gist (geom); +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN + CREATE INDEX IF NOT EXISTS idx_geom_gin ON geometrylog USING gist (geom); + END IF; +END $$; -- Commit the transaction COMMIT; diff --git a/src/backend/migrations/004-project-extra-fields.sql b/src/backend/migrations/004-project-extra-fields.sql new file mode 100644 index 000000000..2aa357216 --- /dev/null +++ b/src/backend/migrations/004-project-extra-fields.sql @@ -0,0 +1,54 @@ +-- ## Migration add some extra fields. +-- * Add geo_restrict_distance_meters and geo_restrict_force_error to projects. +-- * Add new_geom_type to projects. +-- * Add associated_email to organisations. + +-- Related issues: +-- https://github.com/hotosm/fmtm/issues/1985#issuecomment-2577342507 +-- https://github.com/hotosm/fmtm/issues/1979 +-- https://github.com/hotosm/fmtm/issues/2066 + +-- Start a transaction + +BEGIN; + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomtype') THEN + CREATE TYPE public.geomtype AS ENUM ('POINT', 'POLYLINE', 'POLYGON'); + END IF; +END $$; +ALTER TYPE public.geomstatus OWNER TO fmtm; + +ALTER TABLE public.organisations +ADD COLUMN IF NOT EXISTS associated_email +character varying; + +ALTER TABLE public.projects +ADD COLUMN IF NOT EXISTS new_geom_type +public.geomtype +DEFAULT 'POINT', +ADD COLUMN IF NOT EXISTS geo_restrict_distance_meters +int2 +DEFAULT 50 +CHECK (geo_restrict_distance_meters >= 0), +ADD COLUMN IF NOT EXISTS geo_restrict_force_error +BOOLEAN +DEFAULT false; + +-- Also update the fields from previous migration +DROP INDEX IF EXISTS idx_geometrylog; +DROP INDEX IF EXISTS idx_geom_gin; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN + ALTER TABLE public.geometrylog RENAME COLUMN geom TO geojson; + ALTER TABLE public.geometrylog ALTER COLUMN geojson TYPE JSONB USING geojson::jsonb; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_geometrylog_geojson +ON geometrylog USING gin (geojson); + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 64297da72..10c060a4d 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -159,6 +159,13 @@ CREATE TYPE public.geomstatus AS ENUM ( ); ALTER TYPE public.geomstatus OWNER TO fmtm; +CREATE TYPE public.geomtype AS ENUM ( + 'POINT', + 'POLYLINE', + 'POLYGON' +); +ALTER TYPE public.geomtype OWNER TO fmtm; + -- Extra SET default_tablespace = ''; @@ -217,6 +224,7 @@ CREATE TABLE public.organisations ( type public.organisationtype DEFAULT 'FREE', community_type public.communitytype DEFAULT 'OSM_COMMUNITY', created_by integer, + associated_email character varying, approved BOOLEAN DEFAULT false, odk_central_url character varying, odk_central_user character varying, @@ -269,6 +277,11 @@ CREATE TABLE public.projects ( task_num_buildings smallint, hashtags character varying [], custom_tms_url character varying, + geo_restrict_force_error boolean DEFAULT false, + geo_restrict_distance_meters int2 DEFAULT 50 CHECK ( + geo_restrict_distance_meters >= 0 + ), + new_geom_type public.geomtype DEFAULT 'POINT', created_at timestamp with time zone NOT NULL DEFAULT now(), updated_at timestamp with time zone DEFAULT now() ); @@ -392,8 +405,8 @@ ALTER SEQUENCE public.submission_photos_id_seq OWNED BY public.submission_photos.id; CREATE TABLE geometrylog ( - id uuid DEFAULT gen_random_uuid() PRIMARY KEY, - geom JSONB NOT NULL, + id UUID NOT NULL DEFAULT gen_random_uuid(), + geojson JSONB NOT NULL, status geomstatus, project_id int, task_id int @@ -521,8 +534,9 @@ CREATE INDEX idx_entities_task_id ON public.odk_entities USING btree ( entity_id, task_id ); -CREATE INDEX idx_geometrylog -ON geometrylog USING gist (geom); +CREATE INDEX idx_geometrylog_geojson +ON geometrylog USING gin (geom); + -- Foreign keys diff --git a/src/backend/migrations/revert/004-project-extra-fields.sql b/src/backend/migrations/revert/004-project-extra-fields.sql new file mode 100644 index 000000000..efe78e812 --- /dev/null +++ b/src/backend/migrations/revert/004-project-extra-fields.sql @@ -0,0 +1,17 @@ +-- Start a transaction + +BEGIN; + +ALTER TABLE public.organisations +DROP COLUMN IF EXISTS associated_email; + +ALTER TABLE public.projects +DROP CONSTRAINT IF EXISTS projects_geo_restrict_distance_meters_check, +DROP COLUMN IF EXISTS new_geom_type, +DROP COLUMN IF EXISTS geo_restrict_distance_meters, +DROP COLUMN IF EXISTS geo_restrict_force_error; + +DROP TYPE IF EXISTS public.geomtype; + +-- Commit the transaction +COMMIT;