PXB-3318 : prepare_handle_ren_files(): failed to handle .ren files

Problem:
-------
1. take full backup
2. create table t1 before incremental backup
3. Take incremental backup under gdb and pause at backup_start
4. now rename t1 to t2
5. let it finish
6. prepare full
7. prepare incremental

It happens because tables created between full and incremental are copied as .delta/*.meta files and not as IBD files.
prepare_handle_ren_files() relies on *.ibd scan but this cannot work as the *.delta files are not yet loaded to fil_cache.

Fix:
----
Use the meta_map generated from *.meta scan. Use the space_id from space_id.ren to identify the correct .meta and delta files.
Rename the matched .meta and .delta files to the destination name stored in the .ren file

storage/innobase/xtrabackup/src/xtrabackup.cc

@@ -5713,26 +5713,40 @@ static bool prepare_handle_ren_files(
       ut::free(space_name);
       return false;
     }
-    // rename .delta .meta files as well
-    if (xtrabackup_incremental) {
-      std::string from_path =
-          entry.datadir + OS_PATH_SEPARATOR + std::string{space_name};
-      ;
+  }
+
+  // rename .delta .meta files as well
+  if (xtrabackup_incremental) {
+    auto [exists, meta_file] = is_in_meta_map(source_space_id);
+    if (exists) {
       std::string to_path = entry.datadir + ren_file_content;
 
-      rename_force(from_path + ".ibd.delta", to_path + ".delta");
-      rename_force(from_path + ".ibd.meta", to_path + ".meta");
-    }
-    // delete the .ren file, we don't need it anymore
-    os_file_delete(0, ren_path.c_str());
-    ut::free(oldpath);
-    ut::free(space_name);
+      // create .delta path from .meta
+      std::string delta_file =
+          meta_file.substr(0, strlen(meta_file.c_str()) - strlen(".meta")) +
+          ".delta";
 
-    return true;
+      std::string to_delta(to_path + ".delta");
+      xb::info() << "Renaming incremental delta file from: " << delta_file
+                 << " to: " << to_delta;
+      rename_force(delta_file, to_delta);
+
+      std::string to_meta(to_path + ".meta");
+      xb::info() << "Renaming incremental meta file from: " << meta_file
+                 << " to: " << to_meta;
+      rename_force(meta_file, to_meta);
+    } else if (fil_space == nullptr) {
+      // This means the tablespace is neither found in the fullbackup dir
+      // nor in the inc backup directory as .meta and .delta
+      xb::error() << "prepare_handle_ren_files(): failed to handle "
+                  << ren_path;
+      return false;
+    }
   }
 
-  xb::error() << "prepare_handle_ren_files(): failed to handle " << ren_path;
-  return false;
+  // delete the .ren file, we don't need it anymore
+  os_file_delete(0, ren_path.c_str());
+  return true;
 }
 
 /** Handle .corrupt files. These files should be removed before we do *.ibd scan

storage/innobase/xtrabackup/test/suites/lockless/incremental_rename_2.sh

@@ -0,0 +1,56 @@
+###############################################################################
+# PXB-3318 : prepare_handle_ren_files(): failed to handle .ren files
+###############################################################################
+
+. inc/common.sh
+
+require_debug_pxb_version
+start_server
+
+innodb_wait_for_flush_all
+
+xtrabackup --backup --target-dir=$topdir/backup_base --lock-ddl=REDUCED
+
+innodb_wait_for_flush_all
+
+$MYSQL $MYSQL_ARGS -Ns -e "CREATE TABLE t1(a INT);" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 VALUES (1),(2),(3),(4)" test
+
+xtrabackup --backup --target-dir=$topdir/backup_inc --incremental-basedir=$topdir/backup_base \
+  --debug-sync="ddl_tracker_before_lock_ddl" --lock-ddl=REDUCED \
+  2> >( tee $topdir/backup_inc.log)&
+
+job_pid=$!
+pid_file=$topdir/backup_inc/xtrabackup_debug_sync
+wait_for_xb_to_suspend $pid_file
+xb_pid=`cat $pid_file`
+echo "backup pid is $job_pid"
+
+# Generate redo on table t
+$MYSQL $MYSQL_ARGS -Ns -e "RENAME TABLE t1 to t2" test
+
+# Resume the xtrabackup process
+vlog "Resuming xtrabackup"
+kill -SIGCONT $xb_pid
+run_cmd wait $job_pid
+
+xtrabackup --prepare --apply-log-only --target-dir=$topdir/backup_base
+xtrabackup --prepare --target-dir=$topdir/backup_base --incremental-dir=$topdir/backup_inc
+
+# Ensure two things. t1.ibd shouldn't be present
+FILE=$topdir/backup_base/test/t1.ibd
+[ ! -f $FILE ] || die "$FILE exists. It should have been deleted by prepare. Server has renamed the table to t2 during incremental"
+
+# t2.ibd should be present
+FILE=$topdir/backup_base/test/t2.ibd
+[ -f $FILE ] || die "$FILE doesn't exists. It should have been present after rename from t1 to t2"
+
+record_db_state test
+stop_server
+rm -rf $mysql_datadir/*
+xtrabackup --copy-back --target-dir=$topdir/backup_base
+start_server
+verify_db_state test
+rm -rf $topdir/backup_base
+rm -rf $topdir/backup_inc
+rm $topdir/backup_inc.log