Merge remote-tracking branch 'origin/backup-only-claimed-chunks' into…

… dev

src/main/java/serverutils/ServerUtilitiesConfig.java

@@ -322,6 +322,12 @@ public static class Backups {
         @Config.Comment("Delete backups that have a custom name set through /backup start <name>")
         @Config.DefaultBoolean(true)
         public boolean delete_custom_name_backups;
+
+        @Config.Comment("""
+                Only include claimed chunks in backup.
+                Backups will be much faster and smaller, but any unclaimed chunk will be unrecoverable.""")
+        @Config.DefaultBoolean(false)
+        public boolean only_backup_claimed_chunks;
     }
 
     public static class Login {

src/main/java/serverutils/data/ClaimedChunks.java

@@ -163,6 +163,10 @@ public Collection<ClaimedChunk> getAllChunks() {
         return map.isEmpty() ? Collections.emptyList() : map.values();
     }
 
+    public Set<ChunkDimPos> getAllClaimedPositions() {
+        return map.isEmpty() ? Collections.emptySet() : map.keySet();
+    }
+
     public Set<ClaimedChunk> getTeamChunks(@Nullable ForgeTeam team, OptionalInt dimension, boolean includePending) {
         if (team == null) {
             return Collections.emptySet();

src/main/java/serverutils/lib/math/ChunkDimPos.java

@@ -36,11 +36,8 @@ public ChunkDimPos set(int x, int z, int dim) {
         return this;
     }
 
-    public ChunkDimPos set(long packedPos, int dim) {
-        this.posX = CoordinatePacker.unpackX(packedPos);
-        this.posZ = CoordinatePacker.unpackZ(packedPos);
-        this.dim = dim;
-        return this;
+    public ChunkDimPos set(long packed, int dim) {
+        return set(CoordinatePacker.unpackX(packed), CoordinatePacker.unpackZ(packed), dim);
     }
 
     public boolean equals(Object o) {

src/main/java/serverutils/lib/util/FileUtils.java

@@ -230,4 +230,8 @@ public static String getBaseName(File file) {
             return index == -1 ? name : name.substring(0, index);
         }
     }
+
+    public static String getRelativePath(File dir, File file) {
+        return dir.getName() + File.separator + file.getAbsolutePath().substring(dir.getAbsolutePath().length() + 1);
+    }
 }

src/main/java/serverutils/task/backup/BackupTask.java

@@ -7,7 +7,9 @@
 import java.io.File;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -21,7 +23,9 @@
 import serverutils.ServerUtilities;
 import serverutils.ServerUtilitiesConfig;
 import serverutils.ServerUtilitiesNotifications;
+import serverutils.data.ClaimedChunks;
 import serverutils.lib.data.Universe;
+import serverutils.lib.math.ChunkDimPos;
 import serverutils.lib.math.Ticks;
 import serverutils.lib.util.FileUtils;
 import serverutils.lib.util.ServerUtils;
@@ -30,6 +34,7 @@
 public class BackupTask extends Task {
 
     public static final Pattern BACKUP_NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}(.*)");
+    public static final File BACKUP_TEMP_FOLDER = new File("serverutilities/temp/");
     public static File backupsFolder;
     public static ThreadBackup thread;
     public static boolean hadPlayer = false;
@@ -96,11 +101,17 @@ public void execute(Universe universe) {
 
         File worldDir = DimensionManager.getCurrentSaveRootDirectory();
 
+        Set<ChunkDimPos> backupChunks = new HashSet<>();
+        if (backups.only_backup_claimed_chunks && ClaimedChunks.isActive()) {
+            backupChunks.addAll(ClaimedChunks.instance.getAllClaimedPositions());
+            BACKUP_TEMP_FOLDER.mkdirs();
+        }
+
         if (backups.use_separate_thread) {
-            thread = new ThreadBackup(worldDir, customName);
+            thread = new ThreadBackup(worldDir, customName, backupChunks);
             thread.start();
         } else {
-            ThreadBackup.doBackup(worldDir, customName);
+            ThreadBackup.doBackup(worldDir, customName, backupChunks);
         }
         universe.scheduleTask(new BackupTask(true));
     }
@@ -158,6 +169,8 @@ private void postBackup(Universe universe) {
         }
 
         clearOldBackups();
+        FileUtils.delete(BACKUP_TEMP_FOLDER);
+
         thread = null;
         try {
             MinecraftServer server = ServerUtils.getServer();

src/main/java/serverutils/task/backup/ThreadBackup.java

@@ -3,24 +3,47 @@
 import static serverutils.ServerUtilitiesConfig.backups;
 import static serverutils.ServerUtilitiesNotifications.BACKUP_END1;
 import static serverutils.ServerUtilitiesNotifications.BACKUP_END2;
+import static serverutils.task.backup.BackupTask.BACKUP_TEMP_FOLDER;
 
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.List;
+import java.util.Set;
 import java.util.zip.ZipEntry;
 
+import net.minecraft.nbt.CompressedStreamTools;
+import net.minecraft.nbt.NBTTagCompound;
 import net.minecraft.util.EnumChatFormatting;
 import net.minecraft.util.IChatComponent;
+import net.minecraft.world.WorldServer;
+import net.minecraft.world.chunk.storage.RegionFile;
+import net.minecraft.world.chunk.storage.RegionFileCache;
 
+import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
 import org.apache.commons.io.IOUtils;
 
+import com.gtnewhorizon.gtnhlib.util.CoordinatePacker;
+
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import it.unimi.dsi.fastutil.objects.ObjectSet;
 import serverutils.ServerUtilities;
+import serverutils.ServerUtilitiesConfig;
 import serverutils.ServerUtilitiesNotifications;
+import serverutils.lib.math.ChunkDimPos;
 import serverutils.lib.math.Ticks;
 import serverutils.lib.util.FileUtils;
 import serverutils.lib.util.ServerUtils;
@@ -29,78 +52,67 @@
 public class ThreadBackup extends Thread {
 
     private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
-
     private static long logMillis;
     private final File src0;
     private final String customName;
+    private final Set<ChunkDimPos> chunksToBackup;
     public boolean isDone = false;
 
-    public ThreadBackup(File w, String s) {
-        src0 = w;
-        customName = s;
+    public ThreadBackup(File sourceFile, String backupName, Set<ChunkDimPos> backupChunks) {
+        src0 = sourceFile;
+        customName = backupName;
+        chunksToBackup = backupChunks;
         setPriority(7);
     }
 
     public void run() {
         isDone = false;
-        doBackup(src0, customName);
+        doBackup(src0, customName, chunksToBackup);
         isDone = true;
     }
 
-    public static void doBackup(File src, String customName) {
+    public static void doBackup(File src, String customName, Set<ChunkDimPos> chunks) {
         String outName = (customName.isEmpty() ? DATE_FORMAT.format(Calendar.getInstance().getTime()) : customName)
                 + ".zip";
         File dstFile = null;
         try {
             List<File> files = FileUtils.listTree(src);
-            int allFiles = files.size();
-            ServerUtilities.LOGGER.info("Backing up {} files...", files.size());
             long start = System.currentTimeMillis();
+            logMillis = start + Ticks.SECOND.x(5).millis();
 
             dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, outName));
             try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) {
-
                 if (backups.compression_level == 0) {
                     zaos.setMethod(ZipEntry.STORED);
                 } else {
                     zaos.setLevel(backups.compression_level);
                 }
 
-                logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis();
-                ServerUtilities.LOGGER.info("Backing up {} files!", allFiles);
-
-                for (int i = 0; i < allFiles; i++) {
-                    File file = files.get(i);
-                    String filePath = file.getAbsolutePath();
-                    ZipArchiveEntry entry = new ZipArchiveEntry(
-                            src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1));
+                if (!chunks.isEmpty() && backups.only_backup_claimed_chunks) {
+                    backupRegions(src, files, chunks, zaos);
+                } else {
+                    compressFiles(src, files, zaos);
+                }
 
-                    logProgress(i, allFiles, filePath);
-                    zaos.putArchiveEntry(entry);
-                    try (FileInputStream fis = new FileInputStream(file)) {
-                        IOUtils.copy(fis, zaos);
-                    }
-                    zaos.closeArchiveEntry();
+                String backupSize = FileUtils.getSizeString(dstFile);
+                ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize);
+                ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath());
+
+                if (backups.display_file_size) {
+                    String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder);
+                    ServerUtilitiesNotifications.backupNotification(
+                            BACKUP_END2,
+                            "cmd.backup_end_2",
+                            getDoneTime(start),
+                            (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT)));
+                } else {
+                    ServerUtilitiesNotifications
+                            .backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start));
                 }
             }
-            String backupSize = FileUtils.getSizeString(dstFile);
-            ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize);
-            ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath());
-
-            if (backups.display_file_size) {
-                String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder);
-                ServerUtilitiesNotifications.backupNotification(
-                        BACKUP_END2,
-                        "cmd.backup_end_2",
-                        getDoneTime(start),
-                        (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT)));
-            } else {
-                ServerUtilitiesNotifications.backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start));
-            }
         } catch (Exception e) {
-            IChatComponent c = StringUtils.color(
-                    ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()),
-                    EnumChatFormatting.RED);
+            IChatComponent c = StringUtils
+                    .color(ServerUtilities.lang(null, "cmd.backup_fail", e), EnumChatFormatting.RED);
             ServerUtils.notifyChat(ServerUtils.getServer(), null, c);
             ServerUtilities.LOGGER.error("Error while backing up", e);
 
@@ -110,14 +122,132 @@ public static void doBackup(File src, String customName) {
 
     private static void logProgress(int i, int allFiles, String name) {
         long millis = System.currentTimeMillis();
-        if (i == 0 || millis > logMillis || i == allFiles - 1) {
+        boolean first = i == 0;
+        if (first) {
+            ServerUtilities.LOGGER.info("Backing up {} files...", allFiles);
+        }
+
+        if (first || millis > logMillis || i == allFiles - 1) {
             logMillis = millis + Ticks.SECOND.x(5).millis();
             ServerUtilities.LOGGER
                     .info("[{} | {}%]: {}", i, StringUtils.formatDouble00((i / (double) allFiles) * 100D), name);
         }
     }
 
+    private static void compressFiles(File sourceDir, List<File> files, ZipArchiveOutputStream zaos)
+            throws IOException {
+        int allFiles = files.size();
+        for (int i = 0; i < allFiles; i++) {
+            File file = files.get(i);
+            compressFile(FileUtils.getRelativePath(sourceDir, file), file, zaos, i, allFiles);
+        }
+    }
+
+    private static void compressFile(String entryName, File file, ZipArchiveOutputStream out, int index, int totalFiles)
+            throws IOException {
+        ArchiveEntry entry = new ZipArchiveEntry(file, entryName);
+        logProgress(index, totalFiles, file.getAbsolutePath());
+        out.putArchiveEntry(entry);
+        try (FileInputStream fis = new FileInputStream(file)) {
+            IOUtils.copy(fis, out);
+        }
+        out.closeArchiveEntry();
+
+    }
+
+    private static void backupRegions(File sourceFolder, List<File> files, Set<ChunkDimPos> chunksToBackup,
+            ZipArchiveOutputStream out) throws IOException {
+        Object2ObjectMap<File, ObjectSet<ChunkDimPos>> dimRegionClaims = mapClaimsToRegionFile(chunksToBackup);
+        files.removeIf(f -> f.getName().endsWith(".mca"));
+
+        int index = 0;
+        int savedChunks = 0;
+        int regionFiles = dimRegionClaims.size();
+        int totalFiles = files.size() + regionFiles;
+        for (Object2ObjectMap.Entry<File, ObjectSet<ChunkDimPos>> entry : dimRegionClaims.object2ObjectEntrySet()) {
+            File file = entry.getKey();
+            File dimensionRoot = file.getParentFile().getParentFile();
+            File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName()));
+            RegionFile tempRegion = new RegionFile(tempFile);
+            boolean hasData = false;
+
+            for (ChunkDimPos pos : entry.getValue()) {
+                DataInputStream in = RegionFileCache.getChunkInputStream(dimensionRoot, pos.posX, pos.posZ);
+                if (in == null) continue;
+                savedChunks++;
+                hasData = true;
+                NBTTagCompound tag = CompressedStreamTools.read(in);
+                DataOutputStream tempOut = tempRegion.getChunkDataOutputStream(pos.posX & 31, pos.posZ & 31);
+                CompressedStreamTools.write(tag, tempOut);
+                tempOut.close();
+            }
+
+            tempRegion.close();
+            if (hasData) {
+                compressFile(FileUtils.getRelativePath(sourceFolder, file), tempFile, out, index++, totalFiles);
+            }
+
+            FileUtils.delete(tempFile);
+        }
+
+        for (File file : files) {
+            compressFile(FileUtils.getRelativePath(sourceFolder, file), file, out, index++, totalFiles);
+        }
+
+        ServerUtilities.LOGGER.info("Backed up {} regions containing {} claimed chunks", regionFiles, savedChunks);
+    }
+
+    private static Object2ObjectMap<File, ObjectSet<ChunkDimPos>> mapClaimsToRegionFile(
+            Set<ChunkDimPos> chunksToBackup) {
+        Int2ObjectMap<Long2ObjectMap<ObjectSet<ChunkDimPos>>> regionClaimsByDim = new Int2ObjectOpenHashMap<>();
+        chunksToBackup.forEach(
+                pos -> regionClaimsByDim.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>())
+                        .computeIfAbsent(getRegionFromChunk(pos.posX, pos.posZ), k -> new ObjectOpenHashSet<>())
+                        .add(pos));
+
+        Object2ObjectMap<File, ObjectSet<ChunkDimPos>> regionFilesToBackup = new Object2ObjectOpenHashMap<>();
+        for (WorldServer worldserver : ServerUtils.getServer().worldServers) {
+            if (worldserver == null) continue;
+
+            int dim = worldserver.provider.dimensionId;
+            File regionFolder = new File(worldserver.getChunkSaveLocation(), "region");
+            Long2ObjectMap<ObjectSet<ChunkDimPos>> regionClaims = regionClaimsByDim.get(dim);
+            if (!regionFolder.exists() || regionClaims == null) continue;
+
+            File[] regions = regionFolder.listFiles();
+            if (regions == null) continue;
+
+            for (File file : regions) {
+                int[] coords = getRegionCoords(file);
+                long key = CoordinatePacker.pack(coords[0], 0, coords[1]);
+                ObjectSet<ChunkDimPos> claims = regionClaims.get(key);
+                if (claims == null) {
+                    if (ServerUtilitiesConfig.debugging.print_more_info) {
+                        ServerUtilities.LOGGER.info("Skipping region file {} from dimension {}", file.getName(), dim);
+                    }
+                    continue;
+                }
+                regionFilesToBackup.put(file, claims);
+            }
+        }
+        return regionFilesToBackup;
+    }
+
+    private static int[] getRegionCoords(File f) {
+        String fileName = f.getName();
+        int firstDot = fileName.indexOf('.');
+        int secondDot = fileName.indexOf('.', firstDot + 1);
+
+        int x = Integer.parseInt(fileName.substring(firstDot + 1, secondDot));
+        int z = Integer.parseInt(fileName.substring(secondDot + 1, fileName.lastIndexOf('.')));
+        return new int[] { x, z };
+    }
+
     private static String getDoneTime(long l) {
         return StringUtils.getTimeString(System.currentTimeMillis() - l);
     }
+
+    private static long getRegionFromChunk(int chunkX, int chunkZ) {
+        return CoordinatePacker.pack(chunkX >> 5, 0, chunkZ >> 5);
+    }
 }