diff --git a/common/pom.xml b/common/pom.xml index 65dc84e204..80bc17dfea 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -50,6 +50,18 @@ spring-boot-starter + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${jackson.version} + + + + com.fasterxml.jackson.module + jackson-module-blackbird + ${jackson.version} + + org.springframework spring-tx diff --git a/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java b/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java index 901c435e9e..4486af5c04 100644 --- a/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java +++ b/common/src/main/java/com/box/l10n/mojito/json/ObjectMapper.java @@ -6,8 +6,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.blackbird.BlackbirdModule; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -31,6 +33,18 @@ public ObjectMapper(ObjectMapper objectMapper) { registerGuavaModule(); } + public ObjectMapper(SmileFactory smileFactory) { + super(smileFactory); + registerJavaTimeModule(); + registerGuavaModule(); + // Blackbird module uses bytecode generation to further speed up serialization/deserialization + registerBlackbirdModule(); + } + + private void registerBlackbirdModule() { + registerModule(new BlackbirdModule()); + } + private final void registerJavaTimeModule() { JavaTimeModule javaTimeModule = new JavaTimeModule(); registerModule(javaTimeModule); @@ -68,6 +82,14 @@ public String writeValueAsStringUnchecked(Object value) { } } + public byte[] writeValueAsBytes(Object value) { + try { + return super.writeValueAsBytes(value); + } catch (JsonProcessingException jpe) { + throw new RuntimeException(jpe); + } + } + public T readValueUnchecked(File src, Class valueType) { try { return super.readValue(src, valueType); @@ -124,4 +146,10 @@ public static ObjectMapper withNoFailOnUnknownProperties() { objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); return objectMapper; } + + public static ObjectMapper withSmileEnabled() { + ObjectMapper objectMapper = new ObjectMapper(new SmileFactory()); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } } diff --git a/pom.xml b/pom.xml index 0672ed2760..8124275505 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ true true 1.9.20 + 2.13.5 diff --git a/webapp/pom.xml b/webapp/pom.xml index 0636f979d6..f0b4f75a9a 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -266,7 +266,6 @@ jackson-datatype-jsr310 - org.apache.commons diff --git a/webapp/src/main/java/com/box/l10n/mojito/Application.java b/webapp/src/main/java/com/box/l10n/mojito/Application.java index f892b308b8..3212f1b03b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/Application.java +++ b/webapp/src/main/java/com/box/l10n/mojito/Application.java @@ -76,6 +76,12 @@ public ObjectMapper getObjectMapperFailOnUnknownPropertiesFalse() { return objectMapper; } + @Bean(name = "smile_format_object_mapper") + public ObjectMapper getSmileFormatObjectMapper() { + ObjectMapper objectMapper = ObjectMapper.withSmileEnabled(); + return objectMapper; + } + /** * Configuration Jackson ObjectMapper * diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java index 2abcacdd5a..32ef760e50 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java @@ -18,10 +18,18 @@ public Optional getString(Prefix prefix, String name) { return blobStorage.getString(getFullName(prefix, name)); } + public Optional getBytes(Prefix prefix, String name) { + return blobStorage.getBytes(getFullName(prefix, name)); + } + public void put(Prefix prefix, String name, String content, Retention retention) { blobStorage.put(getFullName(prefix, name), content, retention); } + public void putBytes(Prefix prefix, String name, byte[] content, Retention retention) { + blobStorage.put(getFullName(prefix, name), content, retention); + } + public void delete(Prefix prefix, String name) { blobStorage.delete(getFullName(prefix, name)); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java index 5c65d81e5a..3707b11617 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsCacheBlobStorage.java @@ -13,9 +13,14 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @Component +@ConditionalOnProperty( + name = "l10n.cache.textunit.smile.enabled", + havingValue = "false", + matchIfMissing = true) class TextUnitDTOsCacheBlobStorage { static Logger logger = LoggerFactory.getLogger(TextUnitDTOsCacheBlobStorage.class); @@ -39,9 +44,7 @@ class TextUnitDTOsCacheBlobStorage { public Optional> getTextUnitDTOs(Long assetId, Long localeId) { logger.debug( "Get TextUnitDTOs from Blob Storage for assetId: {}, localeId: {}", assetId, localeId); - Optional asString = - structuredBlobStorage.getString(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId)); - return asString.map(this::convertToListOrEmptyList); + return getTextUnitsFromCache(assetId, localeId); } @Timed("TextUnitDTOsCacheBlobStorage.putTextUnitDTOs") @@ -55,9 +58,7 @@ public void putTextUnitDTOs( TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson = new TextUnitDTOsCacheBlobStorageJson(); textUnitDTOsCacheBlobStorageJson.setTextUnitDTOs(textUnitDTOs); - String asString = objectMapper.writeValueAsStringUnchecked(textUnitDTOsCacheBlobStorageJson); - structuredBlobStorage.put( - TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), asString, Retention.PERMANENT); + writeTextUnitDTOsToCache(assetId, localeId, textUnitDTOsCacheBlobStorageJson); } String getName(Long assetId, Long localeId) { @@ -77,4 +78,19 @@ ImmutableList convertToListOrEmptyList(String s) { return ImmutableList.of(); } } + + Optional> getTextUnitsFromCache(Long assetId, Long localeId) { + Optional asString = + structuredBlobStorage.getString(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId)); + return asString.map(this::convertToListOrEmptyList); + } + + void writeTextUnitDTOsToCache( + Long assetId, + Long localeId, + TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson) { + String asString = objectMapper.writeValueAsStringUnchecked(textUnitDTOsCacheBlobStorageJson); + structuredBlobStorage.put( + TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), asString, Retention.PERMANENT); + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorage.java new file mode 100644 index 0000000000..7d48fc23e0 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorage.java @@ -0,0 +1,53 @@ +package com.box.l10n.mojito.service.tm.textunitdtocache; + +import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.TEXT_UNIT_DTOS_CACHE; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.blobstorage.Retention; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.google.common.collect.ImmutableList; +import java.util.Optional; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(name = "l10n.cache.textunit.smile.enabled", havingValue = "true") +public class TextUnitDTOsSmileCacheBlobStorage extends TextUnitDTOsCacheBlobStorage { + + @Autowired + @Qualifier("smile_format_object_mapper") + ObjectMapper objectMapper; + + String getName(Long assetId, Long localeId) { + return "asset/" + assetId + "/locale/" + localeId + ".smile"; + } + + Optional> getTextUnitsFromCache(Long assetId, Long localeId) { + Optional bytes = + structuredBlobStorage.getBytes(TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId)); + return bytes.map(this::convertToListOrEmptyList); + } + + void writeTextUnitDTOsToCache( + Long assetId, + Long localeId, + TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson) { + byte[] bytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson); + structuredBlobStorage.putBytes( + TEXT_UNIT_DTOS_CACHE, getName(assetId, localeId), bytes, Retention.PERMANENT); + } + + ImmutableList convertToListOrEmptyList(byte[] s) { + try { + return ImmutableList.copyOf( + objectMapper.readValue(s, TextUnitDTOsCacheBlobStorageJson.class).getTextUnitDTOs()); + } catch (Exception e) { + logger.error( + "Can't convert the content into TextUnitDTOsCacheBlobStorageJson, return an empty list instead", + e); + return ImmutableList.of(); + } + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorageTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorageTest.java new file mode 100644 index 0000000000..4a73a931be --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/textunitdtocache/TextUnitDTOsSmileCacheBlobStorageTest.java @@ -0,0 +1,75 @@ +package com.box.l10n.mojito.service.tm.textunitdtocache; + +import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.TEXT_UNIT_DTOS_CACHE; +import static org.junit.Assert.assertEquals; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.blobstorage.Retention; +import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.Optional; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class TextUnitDTOsSmileCacheBlobStorageTest { + + TextUnitDTOsSmileCacheBlobStorage textUnitDTOsCacheBlobStorage = + new TextUnitDTOsSmileCacheBlobStorage(); + + ObjectMapper objectMapper = ObjectMapper.withSmileEnabled(); + + @Before + public void setUp() { + textUnitDTOsCacheBlobStorage.objectMapper = objectMapper; + } + + @Test + public void testSuffixAppendedToName() { + String blobName = textUnitDTOsCacheBlobStorage.getName(1234L, 56L); + assertEquals("asset/1234/locale/56.smile", blobName); + } + + @Test + public void testBytesWrittenToCache() { + StructuredBlobStorage structuredBlobStorageMock = Mockito.mock(StructuredBlobStorage.class); + TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson = + new TextUnitDTOsCacheBlobStorageJson(); + textUnitDTOsCacheBlobStorage.structuredBlobStorage = structuredBlobStorageMock; + byte[] expectedBytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson); + textUnitDTOsCacheBlobStorage.writeTextUnitDTOsToCache( + 1234L, 56L, textUnitDTOsCacheBlobStorageJson); + Mockito.verify(structuredBlobStorageMock) + .putBytes( + TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile", expectedBytes, Retention.PERMANENT); + } + + @Test + public void testGetTextUnitDTOS() throws IOException { + StructuredBlobStorage structuredBlobStorageMock = Mockito.mock(StructuredBlobStorage.class); + TextUnitDTOsCacheBlobStorageJson textUnitDTOsCacheBlobStorageJson = + new TextUnitDTOsCacheBlobStorageJson(); + TextUnitDTO textUnitDTO = new TextUnitDTO(); + textUnitDTO.setComment("testComment"); + textUnitDTO.setTmTextUnitId(1L); + textUnitDTO.setName("testName"); + textUnitDTO.setSource("testSource"); + textUnitDTOsCacheBlobStorageJson.setTextUnitDTOs(ImmutableList.of(textUnitDTO)); + byte[] expectedBytes = objectMapper.writeValueAsBytes(textUnitDTOsCacheBlobStorageJson); + Mockito.when( + structuredBlobStorageMock.getBytes(TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile")) + .thenReturn(Optional.of(expectedBytes)); + textUnitDTOsCacheBlobStorage.structuredBlobStorage = structuredBlobStorageMock; + Optional> textUnitDTOS = + textUnitDTOsCacheBlobStorage.getTextUnitsFromCache(1234L, 56L); + Mockito.verify(structuredBlobStorageMock) + .getBytes(TEXT_UNIT_DTOS_CACHE, "asset/1234/locale/56.smile"); + assertEquals(1, textUnitDTOS.get().size()); + assertEquals("testComment", textUnitDTOS.get().get(0).getComment()); + assertEquals(1L, textUnitDTOS.get().get(0).getTmTextUnitId().longValue()); + assertEquals("testName", textUnitDTOS.get().get(0).getName()); + assertEquals("testSource", textUnitDTOS.get().get(0).getSource()); + } +}