From 83e73716abf62d88f9c8265cf868db16b798fb5e Mon Sep 17 00:00:00 2001 From: Colm Date: Mon, 23 Dec 2024 13:00:37 +0000 Subject: [PATCH] PG17 compatibility: fix pg16 to pg17 upgrade (#7788) In `citus_prepare_upgrade()`, don't drop any_value when upgrading from PG16+, because PG16+ has its own any_value function. Attempting to do so results in the error seen in [pg16-pg17 upgrade](https://github.com/citusdata/citus/actions/runs/11768444117/job/32778340003?pr=7661): ``` ERROR: cannot drop function any_value(anyelement) because it is required by the database system CONTEXT: SQL statement "DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement)" ``` When 16 becomes the minimum supported Postgres version, the drop statements can be removed. --- .../distributed/sql/citus--12.1-1--13.0-1.sql | 1 + .../udfs/citus_prepare_pg_upgrade/13.0-1.sql | 100 ++++++++++++++++++ .../udfs/citus_prepare_pg_upgrade/latest.sql | 14 +-- 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql diff --git a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql index 3a342a0fedf..21617166427 100644 --- a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql @@ -1,3 +1,4 @@ -- citus--12.1-1--13.0-1.sql -- bump version to 13.0-1 +#include "udfs/citus_prepare_pg_upgrade/13.0-1.sql" diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql new file mode 100644 index 00000000000..4f07ce5c4b0 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql @@ -0,0 +1,100 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $cppu$ +BEGIN + + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + -- + -- We are dropping the aggregates because postgres 14 changed + -- array_cat type from anyarray to anycompatiblearray. When + -- upgrading to pg14, specifically when running pg_restore on + -- array_cat_agg we would get an error. So we drop the aggregate + -- and create the right one on citus_finish_pg_upgrade. + + DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); + DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); + + -- We should drop any_value because PG16+ has its own any_value function + -- We can remove this part when we drop support for PG16 + IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); + DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + END IF; + + -- + -- Drop existing backup tables + -- + DROP TABLE IF EXISTS public.pg_dist_partition; + DROP TABLE IF EXISTS public.pg_dist_shard; + DROP TABLE IF EXISTS public.pg_dist_placement; + DROP TABLE IF EXISTS public.pg_dist_node_metadata; + DROP TABLE IF EXISTS public.pg_dist_node; + DROP TABLE IF EXISTS public.pg_dist_local_group; + DROP TABLE IF EXISTS public.pg_dist_transaction; + DROP TABLE IF EXISTS public.pg_dist_colocation; + DROP TABLE IF EXISTS public.pg_dist_authinfo; + DROP TABLE IF EXISTS public.pg_dist_poolinfo; + DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; + DROP TABLE IF EXISTS public.pg_dist_object; + DROP TABLE IF EXISTS public.pg_dist_cleanup; + DROP TABLE IF EXISTS public.pg_dist_schema; + DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; + + -- + -- backup citus catalog tables + -- + CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; + CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; + CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; + CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; + CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; + CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; + CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; + CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; + CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; + -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade + CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; + -- enterprise catalog tables + CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; + CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; + -- sequences + CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; + CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT + name, + default_strategy, + shard_cost_function::regprocedure::text, + node_capacity_function::regprocedure::text, + shard_allowed_on_node_function::regprocedure::text, + default_threshold, + minimum_threshold, + improvement_threshold + FROM pg_catalog.pg_dist_rebalance_strategy; + + -- store upgrade stable identifiers on pg_dist_object catalog + CREATE TABLE public.pg_dist_object AS SELECT + address.type, + address.object_names, + address.object_args, + objects.distribution_argument_index, + objects.colocationid + FROM pg_catalog.pg_dist_object objects, + pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we will need to regenerate the partkeys because they will include varnullingrels as well. + -- so we save the partkeys as column names here + CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS + SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name + FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; +END; +$cppu$; + +COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() + IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql index b4bc653f2f3..4f07ce5c4b0 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql @@ -18,13 +18,15 @@ BEGIN DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); - -- We should drop any_value because PG16 has its own any_value function + -- We should drop any_value because PG16+ has its own any_value function -- We can remove this part when we drop support for PG16 - DELETE FROM pg_depend WHERE - objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND - refobjid IN (select oid from pg_extension where extname = 'citus'); - DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); - DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); + DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + END IF; -- -- Drop existing backup tables