Add unit tests to test removing compression from remote paths. Change…

… the approach accordingly.
Netflix · Oct 4, 2024 · 8aa23c6 · 8aa23c6
1 parent 15041ee
commit 8aa23c6
Show file tree

Hide file tree

Showing 5 changed files with 140 additions and 27 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java
@@ -58,14 +58,6 @@ public enum BackupFileType {
         public static boolean isDataFile(BackupFileType type) {
             return DATA_FILE_TYPES.contains(type);
         }
-
-        public static BackupFileType fromString(String s) throws BackupRestoreException {
-            try {
-                return BackupFileType.valueOf(s);
-            } catch (IllegalArgumentException e) {
-                throw new BackupRestoreException(String.format("Unknown BackupFileType %s", s));
-            }
-        }
     }
 
     protected BackupFileType type;

diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java
@@ -17,7 +17,14 @@
 
 package com.netflix.priam.backupv2;
 
-import com.netflix.priam.backup.*;
+import com.google.api.client.util.Lists;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.netflix.priam.backup.AbstractBackupPath;
+import com.netflix.priam.backup.BackupRestoreException;
+import com.netflix.priam.backup.IBackupFileSystem;
+import com.netflix.priam.backup.Status;
 import com.netflix.priam.compress.CompressionType;
 import com.netflix.priam.config.IBackupRestoreConfig;
 import com.netflix.priam.config.IConfiguration;
@@ -27,6 +34,15 @@
 import com.netflix.priam.scheduler.Task;
 import com.netflix.priam.scheduler.TaskTimer;
 import com.netflix.priam.utils.DateUtil;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.math.Fraction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Provider;
+import javax.inject.Singleton;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
@@ -35,14 +51,6 @@
 import java.util.*;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
-import javax.inject.Inject;
-import javax.inject.Named;
-import javax.inject.Provider;
-import javax.inject.Singleton;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.math.Fraction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * This class is used to TTL or delete the SSTable components from the backups after they are not
@@ -70,6 +78,9 @@ public class BackupTTLTask extends Task {
     private final int BATCH_SIZE = 1000;
     private final Instant start_of_feature = DateUtil.parseInstant("201801010000");
     private final int maxWaitMillis;
+    private static final int BACKUP_TYPE_INDEX = 3;
+    private static final Splitter PATH_SPLITTER = Splitter.on('/');
+    private static final Joiner PATH_JOINER = Joiner.on('/');
 
     @Inject
     public BackupTTLTask(
@@ -269,10 +280,24 @@ public void process(ColumnFamilyResult columnfamilyResult) {
         }
     }
 
-    private String removeCompressionPart(String backupPath) {
-        for (CompressionType compressionType : CompressionType.values()) {
-            backupPath = backupPath.replace("/" + compressionType.name() + "/", "/");
+    static String removeCompressionPart(String backupPath) {
+        List<String> parts = Lists.newArrayList(PATH_SPLITTER.split(backupPath));
+        AbstractBackupPath.BackupFileType fileType =
+            AbstractBackupPath.BackupFileType.valueOf(parts.get(BACKUP_TYPE_INDEX));
+        String compressionType;
+        if (fileType == AbstractBackupPath.BackupFileType.SST_V2) {
+            compressionType = parts.remove(7);
+        } else if (fileType == AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2) {
+            compressionType = parts.remove(8);
+        } else {
+            throw new IllegalStateException(
+                String.format("only %s, and %s, are supported, saw %s",
+                    AbstractBackupPath.BackupFileType.SST_V2.name(),
+                    AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2.name(),
+                    fileType));
         }
-        return backupPath;
+        // check compressionType validity
+        CompressionType.valueOf(compressionType);
+        return PATH_JOINER.join(parts);
     }
 }
diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java
@@ -17,6 +17,7 @@
 package com.netflix.priam.backupv2;
 
 import com.google.gson.stream.JsonReader;
+import com.netflix.priam.backup.BackupRestoreException;
 import com.netflix.priam.utils.DateUtil;
 import com.netflix.priam.utils.GsonJsonSerializer;
 import java.io.FileNotFoundException;

diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java
@@ -162,9 +162,9 @@ private Set<AbstractBackupPath.BackupFileType> getUpdatedNotifiedBackupFileTypes
                 for (String s : this.notifiedBackupFileTypes.split(",")) {
                     try {
                         AbstractBackupPath.BackupFileType backupFileType =
-                                AbstractBackupPath.BackupFileType.fromString(s.trim());
+                                AbstractBackupPath.BackupFileType.valueOf(s.trim());
                         notifiedBackupFileTypesSet.add(backupFileType);
-                    } catch (BackupRestoreException ignored) {
+                    } catch (IllegalArgumentException ignored) {
                     }
                 }
             }

diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java
@@ -18,12 +18,10 @@
 package com.netflix.priam.backupv2;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.truth.Truth;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
-import com.netflix.priam.backup.AbstractBackupPath;
-import com.netflix.priam.backup.BRTestModule;
-import com.netflix.priam.backup.FakeBackupFileSystem;
-import com.netflix.priam.backup.Status;
+import com.netflix.priam.backup.*;
 import com.netflix.priam.config.BackupsToCompress;
 import com.netflix.priam.config.FakeConfiguration;
 import com.netflix.priam.config.IConfiguration;
@@ -46,6 +44,7 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
 
 /** Created by aagrawal on 12/17/18. */
 public class TestBackupTTLTask {
@@ -230,4 +229,100 @@ public void testFileIsNotDeletedIfOnlyCompressionIsDifferent() throws Exception
 
         Assert.assertTrue(getAllFiles().contains(data));
     }
+
+    @Test
+    public void testRemoveCompressionPart_sst_NONE() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/table-9f5c6374d48532299a0a5094af9ad1e3/NONE/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/table-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_SNAPPY() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/table-9f5c6374d48532299a0a5094af9ad1e3/SNAPPY/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/table-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_unsupportedCompression() {
+        Assertions.assertThrows(IllegalArgumentException.class, () -> BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/table-9f5c6374d48532299a0a5094af9ad1e3/LZ4/PLAINTEXT/nb-10-big-Statistics.db"));
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_SNAPPYInKeyspace() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/SNAPPY/table-9f5c6374d48532299a0a5094af9ad1e3/SNAPPY/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/SNAPPY/table-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_NONEInKeyspace() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/NONE/table-9f5c6374d48532299a0a5094af9ad1e3/NONE/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/NONE/table-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_SNAPPYInTable() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/SNAPPY-9f5c6374d48532299a0a5094af9ad1e3/SNAPPY/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/SNAPPY-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_sst_NONEInTable() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/NONE-9f5c6374d48532299a0a5094af9ad1e3/NONE/PLAINTEXT/nb-10-big-Statistics.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SST_V2/1724992379000/keyspace/NONE-9f5c6374d48532299a0a5094af9ad1e3/PLAINTEXT/nb-10-big-Statistics.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_NONE() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/NONE/PLAINTEXT/keyspace-table.table_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/PLAINTEXT/keyspace-table.table_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_SNAPPY() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/SNAPPY/PLAINTEXT/keyspace-table.table_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/PLAINTEXT/keyspace-table.table_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_unsupportedCompression() {
+        Assertions.assertThrows(
+            IllegalArgumentException.class,
+            () -> BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/LZ4/PLAINTEXT/keyspace-table.table_field_idx-ka-1-CRC.db"));
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_SNAPPYInKeyspace() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/SNAPPY/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/SNAPPY/PLAINTEXT/SNAPPY-table.table_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/SNAPPY/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/PLAINTEXT/SNAPPY-table.table_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_NONEInKeyspace() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/NONE/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/NONE/PLAINTEXT/NONE-table.table_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/NONE/table-672cc1f038a311e68d78cd36d8a9052a/.table_field_idx/PLAINTEXT/NONE-table.table_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_SNAPPYInTable() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/SNAPPY-672cc1f038a311e68d78cd36d8a9052a/.SNAPPY_field_idx/SNAPPY/PLAINTEXT/keyspace-SNAPPY.SNAPPY_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/SNAPPY-672cc1f038a311e68d78cd36d8a9052a/.SNAPPY_field_idx/PLAINTEXT/keyspace-SNAPPY.SNAPPY_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_NONEInTable() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/NONE-672cc1f038a311e68d78cd36d8a9052a/.NONE_field_idx/NONE/PLAINTEXT/keyspace-NONE.NONE_field_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/NONE-672cc1f038a311e68d78cd36d8a9052a/.NONE_field_idx/PLAINTEXT/keyspace-NONE.NONE_field_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_SNAPPYInIndex() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_SNAPPY_idx/SNAPPY/PLAINTEXT/keyspace-table.table_SNAPPY_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_SNAPPY_idx/PLAINTEXT/keyspace-table.table_SNAPPY_idx-ka-1-CRC.db");
+    }
+
+    @Test
+    public void testRemoveCompressionPart_si_NONEInIndex() {
+        Truth.assertThat(BackupTTLTask.removeCompressionPart("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_NONE_idx/NONE/PLAINTEXT/keyspace-table.table_NONE_idx-ka-1-CRC.db"))
+            .isEqualTo("test_backup/1234_cass_foo/-4419532432517671141/SECONDARY_INDEX_V2/1724992379000/keyspace/table-672cc1f038a311e68d78cd36d8a9052a/.table_NONE_idx/PLAINTEXT/keyspace-table.table_NONE_idx-ka-1-CRC.db");
+    }
 }