diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index e1e0149d38..1c8a894f94 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -653,6 +653,24 @@ public enum CoreError implements ScalarDbError { "Encrypted columns are not supported in the ScalarDB Community edition", "", ""), + DATA_LOADER_INVALID_COLUMN_NON_EXISTENT( + Category.USER_ERROR, + "0144", + "Invalid key: Column %s does not exist in the table %s in namespace %s.", + "", + ""), + DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE( + Category.USER_ERROR, + "0145", + "Invalid base64 encoding for blob value for column %s in table %s in namespace %s", + "", + ""), + DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE( + Category.USER_ERROR, + "0146", + "Invalid number specified for column %s in table %s in namespace %s", + "", + ""), // // Errors for the concurrency error category diff --git a/data-loader/build.gradle b/data-loader/build.gradle index dc73d45023..26a27c6e40 100644 --- a/data-loader/build.gradle +++ b/data-loader/build.gradle @@ -1,9 +1,11 @@ subprojects { group = "scalardb.dataloader" + ext { apacheCommonsLangVersion = '3.14.0' apacheCommonsIoVersion = '2.16.1' + lombokVersion = '1.18.34' } dependencies { // AssertJ @@ -23,5 +25,12 @@ subprojects { testImplementation "org.mockito:mockito-core:${mockitoVersion}" testImplementation "org.mockito:mockito-inline:${mockitoVersion}" testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}" + + + // Lombok + compileOnly "org.projectlombok:lombok:${lombokVersion}" + annotationProcessor "org.projectlombok:lombok:${lombokVersion}" + testCompileOnly "org.projectlombok:lombok:${lombokVersion}" + testAnnotationProcessor "org.projectlombok:lombok:${lombokVersion}" } } diff --git a/data-loader/core/build.gradle b/data-loader/core/build.gradle index 28a2dba4de..5239fd42ef 100644 --- a/data-loader/core/build.gradle +++ b/data-loader/core/build.gradle @@ -9,7 +9,7 @@ archivesBaseName = "scalardb-data-loader-core" dependencies { // ScalarDB core implementation project(':core') - + // for SpotBugs compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" testCompileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnInfo.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnInfo.java new file mode 100644 index 0000000000..de73d43397 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnInfo.java @@ -0,0 +1,13 @@ +package com.scalar.db.dataloader.core; + +import lombok.Builder; +import lombok.Value; + +/** Represents a column in a table. */ +@Value +@Builder +public class ColumnInfo { + String namespace; + String tableName; + String columnName; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/ColumnParsingException.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/ColumnParsingException.java new file mode 100644 index 0000000000..06495156b3 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/ColumnParsingException.java @@ -0,0 +1,16 @@ +package com.scalar.db.dataloader.core.exception; + +/** + * An exception that is thrown when an error occurs while trying to create a ScalarDB column from a + * value + */ +public class ColumnParsingException extends Exception { + + public ColumnParsingException(String message) { + super(message); + } + + public ColumnParsingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/KeyParsingException.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/KeyParsingException.java new file mode 100644 index 0000000000..0ce428eb64 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/exception/KeyParsingException.java @@ -0,0 +1,15 @@ +package com.scalar.db.dataloader.core.exception; + +/** + * An exception that is thrown when an error occurs while trying to create a ScalarDB from a value + */ +public class KeyParsingException extends Exception { + + public KeyParsingException(String message) { + super(message); + } + + public KeyParsingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java new file mode 100644 index 0000000000..74b8f7cb3d --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java @@ -0,0 +1,83 @@ +package com.scalar.db.dataloader.core.util; + +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.exception.ColumnParsingException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.BooleanColumn; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.DoubleColumn; +import com.scalar.db.io.FloatColumn; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.TextColumn; +import java.util.Base64; +import javax.annotation.Nullable; + +/** Utility class for dealing and creating ScalarDB Columns */ +public final class ColumnUtils { + + /** Restrict instantiation via private constructor */ + private ColumnUtils() {} + + /** + * Create a ScalarDB column from the given data type, column name, and value. Blob source values + * need to be base64 encoded. + * + * @param dataType Data type of the specified column + * @param columnInfo ScalarDB table column information + * @param value Value for the ScalarDB column + * @return ScalarDB column + * @throws ColumnParsingException if an error occurs while creating the column and parsing the + * value + */ + public static Column createColumnFromValue( + DataType dataType, ColumnInfo columnInfo, @Nullable String value) + throws ColumnParsingException { + String columnName = columnInfo.getColumnName(); + try { + switch (dataType) { + case BOOLEAN: + return value != null + ? BooleanColumn.of(columnName, Boolean.parseBoolean(value)) + : BooleanColumn.ofNull(columnName); + case INT: + return value != null + ? IntColumn.of(columnName, Integer.parseInt(value)) + : IntColumn.ofNull(columnName); + case BIGINT: + return value != null + ? BigIntColumn.of(columnName, Long.parseLong(value)) + : BigIntColumn.ofNull(columnName); + case FLOAT: + return value != null + ? FloatColumn.of(columnName, Float.parseFloat(value)) + : FloatColumn.ofNull(columnName); + case DOUBLE: + return value != null + ? DoubleColumn.of(columnName, Double.parseDouble(value)) + : DoubleColumn.ofNull(columnName); + case TEXT: + return value != null ? TextColumn.of(columnName, value) : TextColumn.ofNull(columnName); + case BLOB: + // Source blob values need to be base64 encoded + return value != null + ? BlobColumn.of(columnName, Base64.getDecoder().decode(value)) + : BlobColumn.ofNull(columnName); + default: + throw new AssertionError(); + } + } catch (NumberFormatException e) { + throw new ColumnParsingException( + CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage( + columnName, columnInfo.getTableName(), columnInfo.getNamespace()), + e); + } catch (IllegalArgumentException e) { + throw new ColumnParsingException( + CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage( + columnName, columnInfo.getTableName(), columnInfo.getNamespace()), + e); + } + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java new file mode 100644 index 0000000000..ee6e41cfd6 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/KeyUtils.java @@ -0,0 +1,75 @@ +package com.scalar.db.dataloader.core.util; + +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import com.scalar.db.dataloader.core.exception.ColumnParsingException; +import com.scalar.db.dataloader.core.exception.KeyParsingException; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.Key; +import javax.annotation.Nullable; + +/** Utility class for creating and dealing with ScalarDB keys. */ +public final class KeyUtils { + + /** Restrict instantiation via private constructor */ + private KeyUtils() {} + + /** + * Convert a keyValue, in the format of =, to a ScalarDB Key instance for a specific + * ScalarDB table. + * + * @param columnKeyValue A key value in the format of = + * @param namespace Name of the ScalarDB namespace + * @param tableName Name of the ScalarDB table + * @param tableMetadata Metadata for one ScalarDB table + * @return A new ScalarDB Key instance formatted by data type + * @throws KeyParsingException if there is an error parsing the key value + */ + @Nullable + public static Key parseKeyValue( + @Nullable ColumnKeyValue columnKeyValue, + String namespace, + String tableName, + TableMetadata tableMetadata) + throws KeyParsingException { + if (columnKeyValue == null) { + return null; + } + String columnName = columnKeyValue.getColumnName(); + DataType columnDataType = tableMetadata.getColumnDataType(columnName); + if (columnDataType == null) { + throw new KeyParsingException( + CoreError.DATA_LOADER_INVALID_COLUMN_NON_EXISTENT.buildMessage( + columnName, tableName, namespace)); + } + ColumnInfo columnInfo = + ColumnInfo.builder() + .namespace(namespace) + .tableName(tableName) + .columnName(columnName) + .build(); + return createKey(columnDataType, columnInfo, columnKeyValue.getColumnValue()); + } + + /** + * Create a ScalarDB key based on the provided data type, column name, and value. + * + * @param dataType Data type of the specified column + * @param columnInfo ScalarDB table column information + * @param value Value for ScalarDB key + * @return ScalarDB Key instance + * @throws KeyParsingException if there is an error while creating a ScalarDB key + */ + public static Key createKey(DataType dataType, ColumnInfo columnInfo, String value) + throws KeyParsingException { + try { + Column keyValue = ColumnUtils.createColumnFromValue(dataType, columnInfo, value); + return Key.newBuilder().add(keyValue).build(); + } catch (ColumnParsingException e) { + throw new KeyParsingException(e.getMessage(), e); + } + } +} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java new file mode 100644 index 0000000000..cd47243b16 --- /dev/null +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java @@ -0,0 +1,108 @@ +package com.scalar.db.dataloader.core.util; + +import static org.junit.jupiter.api.Assertions.*; + +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.exception.ColumnParsingException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.BooleanColumn; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.DoubleColumn; +import com.scalar.db.io.FloatColumn; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.TextColumn; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class ColumnUtilsTest { + + private static final float FLOAT_VALUE = 2.78f; + + private static Stream provideColumnsForCreateColumnFromValue() { + return Stream.of( + Arguments.of(DataType.BOOLEAN, "boolColumn", "true", BooleanColumn.of("boolColumn", true)), + Arguments.of(DataType.BOOLEAN, "boolColumn", null, BooleanColumn.ofNull("boolColumn")), + Arguments.of(DataType.INT, "intColumn", "42", IntColumn.of("intColumn", 42)), + Arguments.of(DataType.INT, "intColumn", null, IntColumn.ofNull("intColumn")), + Arguments.of( + DataType.BIGINT, + "bigintColumn", + "123456789012", + BigIntColumn.of("bigintColumn", 123456789012L)), + Arguments.of(DataType.BIGINT, "bigintColumn", null, BigIntColumn.ofNull("bigintColumn")), + Arguments.of( + DataType.FLOAT, + "floatColumn", + Float.toString(FLOAT_VALUE), + FloatColumn.of("floatColumn", FLOAT_VALUE)), + Arguments.of(DataType.FLOAT, "floatColumn", null, FloatColumn.ofNull("floatColumn")), + Arguments.of( + DataType.DOUBLE, + "doubleColumn", + Double.toString(Math.E), + DoubleColumn.of("doubleColumn", Math.E)), + Arguments.of(DataType.DOUBLE, "doubleColumn", null, DoubleColumn.ofNull("doubleColumn")), + Arguments.of( + DataType.TEXT, + "textColumn", + "Hello, world!", + TextColumn.of("textColumn", "Hello, world!")), + Arguments.of(DataType.TEXT, "textColumn", null, TextColumn.ofNull("textColumn")), + Arguments.of( + DataType.BLOB, + "blobColumn", + Base64.getEncoder().encodeToString("binary".getBytes(StandardCharsets.UTF_8)), + BlobColumn.of("blobColumn", "binary".getBytes(StandardCharsets.UTF_8))), + Arguments.of(DataType.BLOB, "blobColumn", null, BlobColumn.ofNull("blobColumn"))); + } + + @ParameterizedTest + @MethodSource("provideColumnsForCreateColumnFromValue") + void createColumnFromValue_validInput_returnsColumn( + DataType dataType, String columnName, String value, Column expectedColumn) + throws ColumnParsingException { + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Column actualColumn = ColumnUtils.createColumnFromValue(dataType, columnInfo, value); + assertEquals(expectedColumn, actualColumn); + } + + @Test + void createColumnFromValue_invalidNumberFormat_throwsNumberFormatException() { + String columnName = "intColumn"; + String value = "not_a_number"; + ColumnInfo columnInfo = + ColumnInfo.builder().namespace("ns").tableName("table").columnName(columnName).build(); + ColumnParsingException exception = + assertThrows( + ColumnParsingException.class, + () -> ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, value)); + assertEquals( + CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage( + columnName, "table", "ns"), + exception.getMessage()); + } + + @Test + void createColumnFromValue_invalidBase64_throwsBase64Exception() { + String columnName = "blobColumn"; + String value = "invalid_base64"; + ColumnInfo columnInfo = + ColumnInfo.builder().namespace("ns").tableName("table").columnName(columnName).build(); + ColumnParsingException exception = + assertThrows( + ColumnParsingException.class, + () -> ColumnUtils.createColumnFromValue(DataType.BLOB, columnInfo, value)); + assertEquals( + CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage( + columnName, "table", "ns"), + exception.getMessage()); + } +} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java new file mode 100644 index 0000000000..f2fe680490 --- /dev/null +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/KeyUtilsTest.java @@ -0,0 +1,149 @@ +package com.scalar.db.dataloader.core.util; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.ColumnKeyValue; +import com.scalar.db.dataloader.core.exception.KeyParsingException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.BooleanColumn; +import com.scalar.db.io.DataType; +import com.scalar.db.io.DoubleColumn; +import com.scalar.db.io.FloatColumn; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.Key; +import com.scalar.db.io.TextColumn; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class KeyUtilsTest { + + @Mock private TableMetadata tableMetadata; + + @Test + void parseKeyValue_nullKeyValue_returnsNull() throws KeyParsingException { + assertNull(KeyUtils.parseKeyValue(null, null, null, tableMetadata)); + } + + @Test + void parseKeyValue_invalidColumnName_throwsKeyParsingException() { + String columnName = "invalidColumn"; + ColumnKeyValue keyValue = new ColumnKeyValue(columnName, "value"); + when(tableMetadata.getColumnDataType(columnName)).thenReturn(null); + + KeyParsingException exception = + assertThrows( + KeyParsingException.class, + () -> KeyUtils.parseKeyValue(keyValue, "namespace", "table", tableMetadata)); + assertEquals( + CoreError.DATA_LOADER_INVALID_COLUMN_NON_EXISTENT.buildMessage( + columnName, "table", "namespace"), + exception.getMessage()); + } + + @Test + void parseKeyValue_validKeyValue_returnsKey() throws KeyParsingException { + String columnName = "columnName"; + String value = "value"; + ColumnKeyValue keyValue = new ColumnKeyValue(columnName, value); + DataType dataType = DataType.TEXT; + when(tableMetadata.getColumnDataType(columnName)).thenReturn(dataType); + + Key expected = Key.newBuilder().add(TextColumn.of(columnName, value)).build(); + Key actual = KeyUtils.parseKeyValue(keyValue, "namespace", "table", tableMetadata); + + assertEquals(expected, actual); + } + + @Test + void createKey_boolean_returnsKey() throws KeyParsingException { + String columnName = "booleanColumn"; + String value = "true"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(BooleanColumn.of(columnName, true)).build(); + Key actual = KeyUtils.createKey(DataType.BOOLEAN, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_int_returnsKey() throws KeyParsingException { + String columnName = "intColumn"; + String value = "42"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(IntColumn.of(columnName, 42)).build(); + Key actual = KeyUtils.createKey(DataType.INT, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_bigint_returnsKey() throws KeyParsingException { + String columnName = "bigintColumn"; + String value = "123456789012345"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(BigIntColumn.of(columnName, 123456789012345L)).build(); + Key actual = KeyUtils.createKey(DataType.BIGINT, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_float_returnsKey() throws KeyParsingException { + String columnName = "floatColumn"; + String value = "1.23"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(FloatColumn.of(columnName, 1.23f)).build(); + Key actual = KeyUtils.createKey(DataType.FLOAT, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_double_returnsKey() throws KeyParsingException { + String columnName = "doubleColumn"; + String value = "1.23"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(DoubleColumn.of(columnName, 1.23)).build(); + Key actual = KeyUtils.createKey(DataType.DOUBLE, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_text_returnsKey() throws KeyParsingException { + String columnName = "textColumn"; + String value = "Hello, world!"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + Key expected = Key.newBuilder().add(TextColumn.of(columnName, value)).build(); + Key actual = KeyUtils.createKey(DataType.TEXT, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_blob_returnsKey() throws KeyParsingException { + String columnName = "blobColumn"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + String value = + Base64.getEncoder().encodeToString("Hello, world!".getBytes(StandardCharsets.UTF_8)); + Key expected = + Key.newBuilder() + .add(BlobColumn.of(columnName, "Hello, world!".getBytes(StandardCharsets.UTF_8))) + .build(); + Key actual = KeyUtils.createKey(DataType.BLOB, columnInfo, value); + assertEquals(expected, actual); + } + + @Test + void createKey_invalidBase64_throwsBase64Exception() { + String columnName = "blobColumn"; + String value = "invalidBase64"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + assertThrows( + KeyParsingException.class, () -> KeyUtils.createKey(DataType.BLOB, columnInfo, value)); + } +}