Adding support for jackson smile data format with blackbird module fo…

…r increased serlization/deserialization performance (#100)
box · Mar 5, 2024 · fb7ff89 · fb7ff89
1 parent 7f3f019
commit fb7ff89
Show file tree

Hide file tree

Showing 9 changed files with 205 additions and 7 deletions.
diff --git a/common/pom.xml b/common/pom.xml
@@ -50,6 +50,18 @@
             <artifactId>spring-boot-starter</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-smile</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-blackbird</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-tx</artifactId>

diff --git a/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java b/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java
@@ -6,8 +6,10 @@
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.fasterxml.jackson.datatype.guava.GuavaModule;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.fasterxml.jackson.module.blackbird.BlackbirdModule;
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -31,6 +33,18 @@ public ObjectMapper(ObjectMapper objectMapper) {
     registerGuavaModule();
   }
 
+  public ObjectMapper(SmileFactory smileFactory) {
+    super(smileFactory);
+    registerJavaTimeModule();
+    registerGuavaModule();
+    // Blackbird module uses bytecode generation to further speed up serialization/deserialization
+    registerBlackbirdModule();
+  }
+
+  private void registerBlackbirdModule() {
+    registerModule(new BlackbirdModule());
+  }
+
   private final void registerJavaTimeModule() {
     JavaTimeModule javaTimeModule = new JavaTimeModule();
     registerModule(javaTimeModule);
@@ -68,6 +82,14 @@ public String writeValueAsStringUnchecked(Object value) {
     }
   }
 
+  public byte[] writeValueAsBytes(Object value) {
+    try {
+      return super.writeValueAsBytes(value);
+    } catch (JsonProcessingException jpe) {
+      throw new RuntimeException(jpe);
+    }
+  }
+
   public <T> T readValueUnchecked(File src, Class<T> valueType) {
     try {
       return super.readValue(src, valueType);
@@ -124,4 +146,10 @@ public static ObjectMapper withNoFailOnUnknownProperties() {
     objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     return objectMapper;
   }
+
+  public static ObjectMapper withSmileEnabled() {
+    ObjectMapper objectMapper = new ObjectMapper(new SmileFactory());
+    objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    return objectMapper;
+  }
 }
diff --git a/pom.xml b/pom.xml
@@ -38,6 +38,7 @@
         <docker.compose.detached.mode>true</docker.compose.detached.mode>
         <docker.compose.remove.volumes>true</docker.compose.remove.volumes>
         <aspectj.version>1.9.20</aspectj.version>
+        <jackson.version>2.13.5</jackson.version>
     </properties>
 
     <dependencies>

diff --git a/webapp/pom.xml b/webapp/pom.xml
@@ -266,7 +266,6 @@
             <artifactId>jackson-datatype-jsr310</artifactId>
         </dependency>
 
-
         <!-- TODO(P1) to be removed? -->
         <dependency>
             <groupId>org.apache.commons</groupId>

diff --git a/webapp/src/main/java/com/box/l10n/mojito/Application.java b/webapp/src/main/java/com/box/l10n/mojito/Application.java
@@ -76,6 +76,12 @@ public ObjectMapper getObjectMapperFailOnUnknownPropertiesFalse() {
     return objectMapper;
   }
 
+  @Bean(name = "smile_format_object_mapper")
+  public ObjectMapper getSmileFormatObjectMapper() {
+    ObjectMapper objectMapper = ObjectMapper.withSmileEnabled();
+    return objectMapper;
+  }
+
   /**
    * Configuration Jackson ObjectMapper
    *

diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java
@@ -18,10 +18,18 @@ public Optional<String> getString(Prefix prefix, String name) {
     return blobStorage.getString(getFullName(prefix, name));
   }
 
+  public Optional<byte[]> getBytes(Prefix prefix, String name) {
+    return blobStorage.getBytes(getFullName(prefix, name));
+  }
+
   public void put(Prefix prefix, String name, String content, Retention retention) {
     blobStorage.put(getFullName(prefix, name), content, retention);
   }
 
+  public void putBytes(Prefix prefix, String name, byte[] content, Retention retention) {
+    blobStorage.put(getFullName(prefix, name), content, retention);
+  }
+
   public void delete(Prefix prefix, String name) {
     blobStorage.delete(getFullName(prefix, name));
   }

diff --git a/...in/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java b/...in/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java
@@ -13,9 +13,14 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Component;
 
 @Component
+@ConditionalOnProperty(
+    name = "l10n.cache.textunit.smile.enabled",
+    havingValue = "false",
+    matchIfMissing = true)
 class TextUnitDTOsCacheBlobStorage {
 
   static Logger logger = LoggerFactory.getLogger(TextUnitDTOsCacheBlobStorage.class);
@@ -39,9 +44,7 @@ class TextUnitDTOsCacheBlobStorage {
   public Optional<ImmutableList<TextUnitDTO>> getTextUnitDTOs(Long assetId, Long localeId) {
     logger.debug(
         "Get TextUnitDTOs from Blob Storage for assetId: {}, localeId: {}", assetId, localeId);
-    Optional<String> asString =
-        structuredBlobStorage.getString(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId));
-    return asString.map(this::convertToListOrEmptyList);
+    return getTextUnitsFromCache(assetId, localeId);
   }
 
   @Timed("TextUnitDTOsCacheBlobStorage.putTextUnitDTOs")
@@ -55,9 +58,7 @@ public void putTextUnitDTOs(
     TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson =
         new TextUnitDTOsCacheBlobStorageJson();
     textUnitDTOsCacheBlobStorageJson.setTextUnitDTOs(textUnitDTOs);
-    String asString = objectMapper.writeValueAsStringUnchecked(textUnitDTOsCacheBlobStorageJson);
-    structuredBlobStorage.put(
-        TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), asString, Retention.PERMANENT);
+    writeTextUnitDTOsToCache(assetId, localeId, textUnitDTOsCacheBlobStorageJson);
   }
 
   String getName(Long assetId, Long localeId) {
@@ -77,4 +78,19 @@ ImmutableList<TextUnitDTO> convertToListOrEmptyList(String s) {
       return ImmutableList.of();
     }
   }
+
+  Optional<ImmutableList<TextUnitDTO>> getTextUnitsFromCache(Long assetId, Long localeId) {
+    Optional<String> asString =
+        structuredBlobStorage.getString(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId));
+    return asString.map(this::convertToListOrEmptyList);
+  }
+
+  void writeTextUnitDTOsToCache(
+      Long assetId,
+      Long localeId,
+      TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson) {
+    String asString = objectMapper.writeValueAsStringUnchecked(textUnitDTOsCacheBlobStorageJson);
+    structuredBlobStorage.put(
+        TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), asString, Retention.PERMANENT);
+  }
 }
diff --git a/...va/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorage.java b/...va/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorage.java
@@ -0,0 +1,53 @@
+package com.box.l10n.mojito.service.tm.textunitdtocache;
+
+import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.TEXT_UNIT_DTOS_CACHE;
+
+import com.box.l10n.mojito.json.ObjectMapper;
+import com.box.l10n.mojito.service.blobstorage.Retention;
+import com.box.l10n.mojito.service.tm.search.TextUnitDTO;
+import com.google.common.collect.ImmutableList;
+import java.util.Optional;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnProperty(name = "l10n.cache.textunit.smile.enabled", havingValue = "true")
+public class TextUnitDTOsSmileCacheBlobStorage extends TextUnitDTOsCacheBlobStorage {
+
+  @Autowired
+  @Qualifier("smile_format_object_mapper")
+  ObjectMapper objectMapper;
+
+  String getName(Long assetId, Long localeId) {
+    return "asset/" + assetId + "/locale/" + localeId + ".smile";
+  }
+
+  Optional<ImmutableList<TextUnitDTO>> getTextUnitsFromCache(Long assetId, Long localeId) {
+    Optional<byte[]> bytes =
+        structuredBlobStorage.getBytes(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId));
+    return bytes.map(this::convertToListOrEmptyList);
+  }
+
+  void writeTextUnitDTOsToCache(
+      Long assetId,
+      Long localeId,
+      TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson) {
+    byte[] bytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson);
+    structuredBlobStorage.putBytes(
+        TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), bytes, Retention.PERMANENT);
+  }
+
+  ImmutableList<TextUnitDTO> convertToListOrEmptyList(byte[] s) {
+    try {
+      return ImmutableList.copyOf(
+          objectMapper.readValue(s, TextUnitDTOsCacheBlobStorageJson.class).getTextUnitDTOs());
+    } catch (Exception e) {
+      logger.error(
+          "Can't convert the content into TextUnitDTOsCacheBlobStorageJson, return an empty list instead",
+          e);
+      return ImmutableList.of();
+    }
+  }
+}
diff --git a/...om/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorageTest.java b/...om/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorageTest.java
@@ -0,0 +1,75 @@
+package com.box.l10n.mojito.service.tm.textunitdtocache;
+
+import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.TEXT_UNIT_DTOS_CACHE;
+import static org.junit.Assert.assertEquals;
+
+import com.box.l10n.mojito.json.ObjectMapper;
+import com.box.l10n.mojito.service.blobstorage.Retention;
+import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage;
+import com.box.l10n.mojito.service.tm.search.TextUnitDTO;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.Optional;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TextUnitDTOsSmileCacheBlobStorageTest {
+
+  TextUnitDTOsSmileCacheBlobStorage textUnitDTOsCacheBlobStorage =
+      new TextUnitDTOsSmileCacheBlobStorage();
+
+  ObjectMapper objectMapper = ObjectMapper.withSmileEnabled();
+
+  @Before
+  public void setUp() {
+    textUnitDTOsCacheBlobStorage.objectMapper = objectMapper;
+  }
+
+  @Test
+  public void testSuffixAppendedToName() {
+    String blobName = textUnitDTOsCacheBlobStorage.getName(1234L, 56L);
+    assertEquals("asset/1234/locale/56.smile", blobName);
+  }
+
+  @Test
+  public void testBytesWrittenToCache() {
+    StructuredBlobStorage structuredBlobStorageMock = Mockito.mock(StructuredBlobStorage.class);
+    TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson =
+        new TextUnitDTOsCacheBlobStorageJson();
+    textUnitDTOsCacheBlobStorage.structuredBlobStorage = structuredBlobStorageMock;
+    byte[] expectedBytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson);
+    textUnitDTOsCacheBlobStorage.writeTextUnitDTOsToCache(
+        1234L, 56L, textUnitDTOsCacheBlobStorageJson);
+    Mockito.verify(structuredBlobStorageMock)
+        .putBytes(
+            TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile", expectedBytes, Retention.PERMANENT);
+  }
+
+  @Test
+  public void testGetTextUnitDTOS() throws IOException {
+    StructuredBlobStorage structuredBlobStorageMock = Mockito.mock(StructuredBlobStorage.class);
+    TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson =
+        new TextUnitDTOsCacheBlobStorageJson();
+    TextUnitDTO textUnitDTO = new TextUnitDTO();
+    textUnitDTO.setComment("testComment");
+    textUnitDTO.setTmTextUnitId(1L);
+    textUnitDTO.setName("testName");
+    textUnitDTO.setSource("testSource");
+    textUnitDTOsCacheBlobStorageJson.setTextUnitDTOs(ImmutableList.of(textUnitDTO));
+    byte[] expectedBytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson);
+    Mockito.when(
+            structuredBlobStorageMock.getBytes(TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile"))
+        .thenReturn(Optional.of(expectedBytes));
+    textUnitDTOsCacheBlobStorage.structuredBlobStorage = structuredBlobStorageMock;
+    Optional<ImmutableList<TextUnitDTO>> textUnitDTOS =
+        textUnitDTOsCacheBlobStorage.getTextUnitsFromCache(1234L, 56L);
+    Mockito.verify(structuredBlobStorageMock)
+        .getBytes(TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile");
+    assertEquals(1, textUnitDTOS.get().size());
+    assertEquals("testComment", textUnitDTOS.get().get(0).getComment());
+    assertEquals(1L, textUnitDTOS.get().get(0).getTmTextUnitId().longValue());
+    assertEquals("testName", textUnitDTOS.get().get(0).getName());
+    assertEquals("testSource", textUnitDTOS.get().get(0).getSource());
+  }
+}