From da827b4a37455664f25a261382fd18bf9d9eb0e8 Mon Sep 17 00:00:00 2001 From: Felix Wu Date: Tue, 26 Apr 2022 11:54:45 -0700 Subject: [PATCH 1/3] catch setting role exception and output a warning instead of stop migration process --- aiven_db_migrate/migrate/pgmigrate.py | 29 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/aiven_db_migrate/migrate/pgmigrate.py b/aiven_db_migrate/migrate/pgmigrate.py index 4984b10..d148d74 100644 --- a/aiven_db_migrate/migrate/pgmigrate.py +++ b/aiven_db_migrate/migrate/pgmigrate.py @@ -992,18 +992,23 @@ def _migrate_roles(self) -> Dict[str, PGRoleTask]: message=err.diag.message_primary, ) else: - if role.rolconfig: - for conf in role.rolconfig: - key, value = conf.split("=", 1) - self.log.info("Setting config for role %r: %s = %s", role.rolname, key, value) - self.target.c(f'ALTER ROLE {role.safe_rolname} SET "{key}" = %s', args=(value, ), return_rows=0) - roles[role.rolname] = PGRoleTask( - rolname=rolname, - rolpassword=role.rolpassword, - status=PGRoleStatus.created, - message="role created", - ) - + try: + if role.rolconfig: + for conf in role.rolconfig: + key, value = conf.split("=", 1) + self.log.info("Setting config for role %r: %s = %s", role.rolname, key, value) + self.target.c(f'ALTER ROLE {role.safe_rolname} SET "{key}" = %s', args=(value, ), return_rows=0) + roles[role.rolname] = PGRoleTask( + rolname=rolname, + rolpassword=role.rolpassword, + status=PGRoleStatus.created, + message="role created", + ) + # display warning when ProgrammingErrorERROR 42501: InsufficientPrivilege: permission denied to set parameter for a role + except psycopg2.errors.InsufficientPrivilege: + self.log.warning( + f'Setting [{role.rolname}]: [{key}] = [{value}] failed. psycopg2.errors.InsufficientPrivilege' + ) return roles @staticmethod From 2759321b9778720499dbede18f7c7fa21e62421b Mon Sep 17 00:00:00 2001 From: Felix Wu Date: Tue, 26 Apr 2022 15:54:16 -0700 Subject: [PATCH 2/3] add -fr , --filtered-roles flag to filter roles not to be migrated --- aiven_db_migrate/migrate/pgmigrate.py | 36 +++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/aiven_db_migrate/migrate/pgmigrate.py b/aiven_db_migrate/migrate/pgmigrate.py index d148d74..2f81059 100644 --- a/aiven_db_migrate/migrate/pgmigrate.py +++ b/aiven_db_migrate/migrate/pgmigrate.py @@ -108,7 +108,13 @@ class PGCluster: _pg_roles: Dict[str, PGRole] _mangle: bool - def __init__(self, conn_info: Union[str, Dict[str, Any]], filtered_db: Optional[str] = None, mangle: bool = False): + def __init__( + self, + conn_info: Union[str, Dict[str, Any]], + filtered_db: Optional[str] = None, + filtered_roles: Optional[str] = None, + mangle: bool = False + ): self.log = logging.getLogger(self.__class__.__name__) self.conn_info = get_connection_info(conn_info) self.conn_lock = threading.RLock() @@ -125,6 +131,10 @@ def __init__(self, conn_info: Union[str, Dict[str, Any]], filtered_db: Optional[ self.filtered_db = filtered_db.split(",") else: self.filtered_db = [] + if filtered_roles: + self.filtered_roles = filtered_roles.split(",") + else: + self.filtered_roles = [] if "application_name" not in self.conn_info: self.conn_info["application_name"] = f"aiven-db-migrate/{__version__}" self._mangle = mangle @@ -318,9 +328,16 @@ def pg_lang(self) -> List[Dict[str, Any]]: @property def pg_roles(self) -> Dict[str, PGRole]: + filtered_roles = ["rdstopmgr"] + if self.filtered_roles: + filtered_roles.extend(self.filtered_roles) + roles_list = ",".join(f"'{role}'" for role in filtered_roles) + if not self._pg_roles: # exclude system roles - roles = self.c("SELECT quote_ident(rolname) as safe_rolname, * FROM pg_catalog.pg_roles WHERE oid > 16384") + roles = self.c( + f"SELECT quote_ident(rolname) as safe_rolname, * FROM pg_catalog.pg_roles WHERE oid > 16384 AND rolname NOT IN ({roles_list})" + ) for r in roles: rolname = r["rolname"] # create semi-random placeholder password for role with login @@ -738,6 +755,7 @@ def __init__( verbose: bool = False, mangle: bool = False, filtered_db: Optional[str] = None, + filtered_roles: Optional[str] = None, skip_tables: Optional[List[str]] = None, with_tables: Optional[List[str]] = None, replicate_extensions: bool = True, @@ -745,8 +763,12 @@ def __init__( if skip_tables and with_tables: raise Exception("Can only specify a skip table list or a with table list") self.log = logging.getLogger(self.__class__.__name__) - self.source = PGSource(conn_info=source_conn_info, filtered_db=filtered_db, mangle=mangle) - self.target = PGTarget(conn_info=target_conn_info, filtered_db=filtered_db, mangle=mangle) + self.source = PGSource( + conn_info=source_conn_info, filtered_db=filtered_db, filtered_roles=filtered_roles, mangle=mangle + ) + self.target = PGTarget( + conn_info=target_conn_info, filtered_db=filtered_db, filtered_roles=filtered_roles, mangle=mangle + ) self.skip_tables = self._convert_table_names(skip_tables) self.with_tables = self._convert_table_names(with_tables) self.pgbin = Path() @@ -1007,7 +1029,7 @@ def _migrate_roles(self) -> Dict[str, PGRoleTask]: # display warning when ProgrammingErrorERROR 42501: InsufficientPrivilege: permission denied to set parameter for a role except psycopg2.errors.InsufficientPrivilege: self.log.warning( - f'Setting [{role.rolname}]: [{key}] = [{value}] failed. psycopg2.errors.InsufficientPrivilege' + f'Setting [{role.rolname}]: [{key}] = [{value}] failed. psycopg2.errors.InsufficientPrivilege' ) return roles @@ -1268,6 +1290,9 @@ def main(args=None, *, prog="pg_migrate"): parser.add_argument( "-t", "--target", help="Target PostgreSQL server, either postgres:// uri or libpq connection string.", required=True ) + parser.add_argument( + "-fr", "--filtered-roles", help="Comma separated list of roles to filter out during migrations", required=False + ) parser.add_argument( "-f", "--filtered-db", help="Comma separated list of databases to filter out during migrations", required=False ) @@ -1356,6 +1381,7 @@ def main(args=None, *, prog="pg_migrate"): stop_replication=args.stop_replication, verbose=args.verbose, filtered_db=args.filtered_db, + filtered_roles=args.filtered_roles, mangle=args.mangle, skip_tables=args.skip_table, with_tables=args.with_table, From 7a98a1cfc6a88d1f86652aeb25cafa8888782c99 Mon Sep 17 00:00:00 2001 From: Felix Wu Date: Wed, 27 Apr 2022 23:36:27 -0700 Subject: [PATCH 3/3] filter roles feature improvements --- aiven_db_migrate/migrate/pgmigrate.py | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/aiven_db_migrate/migrate/pgmigrate.py b/aiven_db_migrate/migrate/pgmigrate.py index 2f81059..efbca29 100644 --- a/aiven_db_migrate/migrate/pgmigrate.py +++ b/aiven_db_migrate/migrate/pgmigrate.py @@ -112,7 +112,7 @@ def __init__( self, conn_info: Union[str, Dict[str, Any]], filtered_db: Optional[str] = None, - filtered_roles: Optional[str] = None, + filtered_roles: Tuple[str] = ((), ), mangle: bool = False ): self.log = logging.getLogger(self.__class__.__name__) @@ -131,10 +131,8 @@ def __init__( self.filtered_db = filtered_db.split(",") else: self.filtered_db = [] - if filtered_roles: - self.filtered_roles = filtered_roles.split(",") - else: - self.filtered_roles = [] + self.filtered_roles = filtered_roles + if "application_name" not in self.conn_info: self.conn_info["application_name"] = f"aiven-db-migrate/{__version__}" self._mangle = mangle @@ -328,18 +326,19 @@ def pg_lang(self) -> List[Dict[str, Any]]: @property def pg_roles(self) -> Dict[str, PGRole]: - filtered_roles = ["rdstopmgr"] - if self.filtered_roles: - filtered_roles.extend(self.filtered_roles) - roles_list = ",".join(f"'{role}'" for role in filtered_roles) if not self._pg_roles: # exclude system roles - roles = self.c( - f"SELECT quote_ident(rolname) as safe_rolname, * FROM pg_catalog.pg_roles WHERE oid > 16384 AND rolname NOT IN ({roles_list})" - ) + roles = self.c(f"SELECT quote_ident(rolname) as safe_rolname, * FROM pg_catalog.pg_roles WHERE oid > 16384") + for r in roles: rolname = r["rolname"] + + if rolname in self.filtered_roles: + self.log.debug(f'Skipping filtered role: [{r["rolname"]}]') + continue + + self.log.debug(f'Migrating role: [{r["rolname"]}]') # create semi-random placeholder password for role with login rolpassword = ( "placeholder_{}".format("".join(random.choices(string.ascii_lowercase, k=16))) @@ -755,7 +754,7 @@ def __init__( verbose: bool = False, mangle: bool = False, filtered_db: Optional[str] = None, - filtered_roles: Optional[str] = None, + filtered_roles: Tuple[str] = ((), ), skip_tables: Optional[List[str]] = None, with_tables: Optional[List[str]] = None, replicate_extensions: bool = True, @@ -1373,6 +1372,9 @@ def main(args=None, *, prog="pg_migrate"): else: logging.basicConfig(level=logging.INFO, format=log_format) + if not args.filtered_roles: + args.filtered_roles = () + pg_mig = PGMigrate( source_conn_info=args.source, target_conn_info=args.target,