From 5953938e224389793929763a1d65f1a005fe942b Mon Sep 17 00:00:00 2001 From: eemjwu <34029771+eemjwu@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:16:13 +0800 Subject: [PATCH 01/11] =?UTF-8?q?=E5=A2=9E=E5=8A=A0same=5Fproperty=5Fnames?= =?UTF-8?q?=E5=88=A4=E6=96=AD=20(#79)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * properties name ignor case --------- Co-authored-by: wumengjie.wmj --- .../impl/execute/ObTableBatchOperation.java | 45 ++++++++-- .../rpc/ObAtomicBatchOperationTest.java | 89 +++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java index 495ffde2..5ccac8f6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java @@ -18,11 +18,11 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; @@ -36,7 +36,7 @@ public class ObTableBatchOperation extends AbstractPayload { private List tableOperations = new ArrayList(); private boolean isReadOnly = true; private boolean isSameType = true; - private boolean isSamePropertiesNames; + private boolean isSamePropertiesNames = true; /* * Encode. @@ -121,6 +121,15 @@ public List getTableOperations() { return tableOperations; } + /* + * hash_map keys to TreeSet IgnoringCase + */ + public TreeSet mapKeysToSetIgnoringCase(Set keys) { + TreeSet keySetIgnoreCase = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + keySetIgnoreCase.addAll(keys); + return keySetIgnoreCase; + } + /* * Add table operation. */ @@ -131,11 +140,22 @@ public void addTableOperation(ObTableOperation tableOperation) { isReadOnly = false; } if (isSameType - && length > 1 - && tableOperations.get(length - 1).getOperationType() != tableOperations + && length > 1 + && tableOperations.get(length - 1).getOperationType() != tableOperations .get(length - 2).getOperationType()) { isSameType = false; } + // 判断是否是 same_properties_name + if (isSamePropertiesNames && length > 1) { + ObTableOperation prev = tableOperations.get(length - 2); + ObTableOperation curr = tableOperations.get(length - 1); + if (prev.getEntity().getPropertiesCount() != curr.getEntity().getPropertiesCount()) { + isSamePropertiesNames = false; + } else { + isSamePropertiesNames = + mapKeysToSetIgnoringCase(prev.getEntity().getProperties().keySet()).equals(mapKeysToSetIgnoringCase(curr.getEntity().getProperties().keySet())); + } + } } /* @@ -145,9 +165,11 @@ public void setTableOperations(List tableOperations) { this.tableOperations = tableOperations; this.isReadOnly = true; this.isSameType = true; + this.isSamePropertiesNames = true; ObTableOperationType prevType = null; + TreeSet firstKeySetIgnoreCase = null; for (ObTableOperation o : tableOperations) { - if (this.isReadOnly || this.isSameType) { + if (this.isReadOnly || this.isSameType || this.isSamePropertiesNames) { if (!o.isReadonly()) { this.isReadOnly = false; } @@ -156,6 +178,17 @@ public void setTableOperations(List tableOperations) { } else { prevType = o.getOperationType(); } + + if (this.isSamePropertiesNames) { + if (firstKeySetIgnoreCase == null) { + firstKeySetIgnoreCase = mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet()); + } else if (firstKeySetIgnoreCase.size() != o.getEntity().getPropertiesCount()) { + this.isSamePropertiesNames = false; + } else { + this.isSamePropertiesNames = + firstKeySetIgnoreCase.equals(mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet())); + } + } } else { return; } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index fdde18ae..758c174c 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -101,4 +101,93 @@ public void testAtomic() { } } + @Test + public void testBatchOperation() { + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + try { + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.insert("abc-1", new String[]{"c1", "c2"}, new String[]{"bar-1", "bar-2"}); + batchOps.get("abc-2", new String[]{"c2"}); + batchOps.insert("abc-3", new String[]{"c2"}, new String[]{"bar-3"}); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); + batchOps.get("abc-4", new String[]{"c1", "c2"}); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); + batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: true + { + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: true + { + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1a", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c2", "c3", "C1A"}); + batchOps.get("abc-4", new String[]{"c1A", "c2", "c3"}); + batchOps.insert("abc-4", new String[]{"c3", "C2", "c1a"}, new String[]{"bar-3", "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c1A", "c2", "C3"}, new String[]{"bar-2", "bar-2", "bar-2"}); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-4", new String[]{"c1", "c2"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-2", "bar-2", "bar-2"}); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + + batchOps.clear(); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c4", "c3"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c2", "c3", "c1"}, new String[]{"bar-2", "bar-2", "bar-2"}); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } + + } + } From 88d94ea6049d49b4501ba5eef63716fb6d0bd431 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:37:05 +0800 Subject: [PATCH 02/11] support put operation (#80) --- .../oceanbase/rpc/ObClusterTableBatchOps.java | 8 + .../alipay/oceanbase/rpc/ObTableClient.java | 41 +++++ .../alipay/oceanbase/rpc/mutation/Put.java | 147 +++++++++++++++++- .../impl/execute/ObTableOperationType.java | 5 +- .../rpc/table/AbstractTableBatchOps.java | 8 + .../rpc/table/ObTableBatchOpsImpl.java | 8 + .../rpc/table/ObTableClientBatchOpsImpl.java | 9 ++ .../table/ObTableClientLSBatchOpsImpl.java | 8 + .../rpc/table/api/TableBatchOps.java | 4 + .../alipay/oceanbase/rpc/ObTablePutTest.java | 79 ++++++++++ src/test/resources/ci.sql | 9 +- 11 files changed, 316 insertions(+), 10 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java index 665b462f..57fa2c76 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java @@ -104,6 +104,14 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean tableBatchOps.append(rowkeys, columns, values, withResult); } + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + tableBatchOps.put(rowkeys, columns, values); + } + /* * Execute. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index a70a7220..16c5b810 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1998,6 +1998,47 @@ public ObPayload execute(ObPair obPair) throws Exception { }); } + /** + * put with result + * @param tableName which table to put + * @param rowKey insert row key + * @param keyRanges scan range + * @param columns columns name to put + * @param values new values + * @return execute result + * @throws Exception exception + */ + public ObPayload putWithResult(final String tableName, final Object[] rowKey, + final List keyRanges, final String[] columns, + final Object[] values) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, PUT, rowKey, columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "PUT", endpoint, rowKey, + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + /** * Replace. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java index a4ddd447..79cf56c5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java @@ -17,20 +17,30 @@ package com.alipay.oceanbase.rpc.mutation; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; import com.alipay.oceanbase.rpc.table.api.Table; -/* - * Put impl. - * Need fill all columns when use put impl. - * Server will do override when use put impl. - */ -public class Put extends InsertOrUpdate { +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Put extends Mutation { + private List columns = null; + private List values = null; + /* * default constructor */ public Put() { super(); - super.usePut(); + columns = new ArrayList(); + values = new ArrayList(); } /* @@ -38,6 +48,127 @@ public Put() { */ public Put(Table client, String tableName) { super(client, tableName); - super.usePut(); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * set the Row Key of mutation with Row and keep scan range + */ + @Override + public Put setRowKey(Row rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * set the Row Key of mutation with ColumnValues and keep scan range + */ + @Override + public Put setRowKey(ColumnValue... rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * add filter into mutation (use QueryAndMutate) and scan range + */ + public Put setFilter(ObTableFilter filter) throws Exception { + return setFilterOnly(filter); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.PUT; + } + + /* + * add mutated Row + */ + public Put addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Put"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Put + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated ColumnValues + */ + public Put addMutateColVal(ColumnValue... columnValues) throws Exception { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Put"); + } + + // set mutate row into Put + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Put removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple Put, without filter + return new MutationResult(((ObTableClient) getClient()).putWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray())); + } else { + if (checkMutationWithFilter()) { + // QueryAndPut + ObTableOperation operation = ObTableOperation.getInstance( + ObTableOperationType.PUT, getRowKey(), columns.toArray(new String[0]), + values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter( + getQuery(), getRowKey(), getKeyRanges(), operation, true)); + } else { + throw new ObTableUnexpectedException("should set filter and scan range both"); + } + } } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java index db6269d2..76faf2e9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java @@ -28,7 +28,10 @@ public enum ObTableOperationType { APPEND(7), // append column value SCAN(8), // query TTL(9), // observer internal type, not used by client - CHECK_AND_INSERT_UP(10), INVALID(11); + CHECK_AND_INSERT_UP(10), + PUT(11), // override row + TRIGGER(12), // internal op type + INVALID(15); private int value; private static Map map = new HashMap(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java index bf9bfaca..1c1c791f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java @@ -92,6 +92,14 @@ public void append(Object rowkey, String[] columns, Object[] values, boolean wit append(new Object[] { rowkey }, columns, values, withResult); } + /** + * Put. + */ + @Override + public void put(Object rowkey, String[] columns, Object[] values) { + put(new Object[] { rowkey }, columns, values); + } + /** * Get table name. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java index 29d7a49d..75432b6a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java @@ -116,6 +116,14 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean addObTableOperation(ObTableOperationType.APPEND, rowkeys, columns, values); } + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + addObTableOperation(ObTableOperationType.PUT, rowkeys, columns, values); + } + /* * Add ob table operation. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index 6b06ea30..5f24d0f3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -142,6 +142,14 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean addObTableClientOperation(ObTableOperationType.APPEND, rowkeys, columns, values); } + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + addObTableClientOperation(ObTableOperationType.PUT, rowkeys, columns, values); + } + private void addObTableClientOperation(ObTableOperationType type, Object[] rowkeys, String[] columns, Object[] values) { ObTableOperation instance = ObTableOperation.getInstance(type, rowkeys, columns, values); @@ -193,6 +201,7 @@ public List executeWithResult() throws Exception { case REPLACE: case INCREMENT: case APPEND: + case PUT: results.add(new MutationResult(result)); break; default: diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index f857c0c0..ea55b77e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -138,6 +138,14 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean throw new FeatureNotSupportedException(); } + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + private void addOperation(ObTableSingleOp singleOp) { batchOperation.add(singleOp); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java index 0ab134cb..0523020c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java @@ -60,6 +60,10 @@ public interface TableBatchOps { void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values); + void put(Object rowkey, String[] columns, Object[] values); + + void put(Object[] rowkeys, String[] columns, Object[] values); + /** * increment the value * @param rowkey the primary key diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java new file mode 100644 index 00000000..50f4f41c --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java @@ -0,0 +1,79 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Timestamp; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ObTablePutTest { + ObTableClient client; + public static String tableName = "test_put"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "id" }); + } + + /** + CREATE TABLE IF NOT EXISTS `test_put` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + `c2` bigint DEFAULT NULL, + `c3` varchar(32) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`id`)) PARTITION BY KEY(`id`) PARTITIONS 32; + **/ + @Test + public void testPut() throws Exception { + try { + client.put(tableName).setRowKey(colVal("id", "id0")).execute(); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("c1", 1L)) + .execute(); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("c3", "c3")) + .execute(); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + client.delete(tableName).setRowKey(colVal("id", "id0")).execute(); + } + + } + +} diff --git a/src/test/resources/ci.sql b/src/test/resources/ci.sql index 668597ef..44988276 100644 --- a/src/test/resources/ci.sql +++ b/src/test/resources/ci.sql @@ -410,7 +410,7 @@ CREATE TABLE IF NOT EXISTS `test_auto_increment_one_rowkey` ( `c1` int auto_increment, `c2` int NOT NULL, PRIMARY KEY(`c1`)); -CREATE TABLE IF NOT EXISTS `sync_item` ( +CREATE TABLE IF NOT EXISTS `sync_item` ( `uid` varchar(20) NOT NULL, `object_id` varchar(32) NOT NULL, `type` int(11) NULL, @@ -446,5 +446,12 @@ CREATE TABLE IF NOT EXISTS `test_table_object` ( `c10` datetime(6) not null, `c11` int default null ); +CREATE TABLE IF NOT EXISTS `test_put` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + `c2` bigint DEFAULT NULL, + `c3` varchar(32) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`id`)) PARTITION BY KEY(`id`) PARTITIONS 32; alter system set kv_hotkey_throttle_threshold = 50; From dd6d1e5d1fcf38ea75416b8b69409fc91214c547 Mon Sep 17 00:00:00 2001 From: eemjwu <34029771+eemjwu@users.noreply.github.com> Date: Wed, 21 Feb 2024 14:00:11 +0800 Subject: [PATCH 03/11] add option flag, batch operations can return one res (#94) --- .../oceanbase/rpc/ObClusterTableBatchOps.java | 6 ++ .../rpc/mutation/BatchOperation.java | 7 +++ .../impl/execute/ObTableBatchOperation.java | 4 +- .../execute/ObTableBatchOperationRequest.java | 6 ++ .../impl/execute/ObTableOptionFlag.java | 12 +++- .../rpc/table/AbstractTableBatchOps.java | 14 +++++ .../rpc/table/ObTableBatchOpsImpl.java | 2 + .../rpc/table/ObTableClientBatchOpsImpl.java | 1 + .../rpc/table/api/TableBatchOps.java | 4 ++ .../rpc/ObAtomicBatchOperationTest.java | 56 +++++++++++++++++++ 10 files changed, 110 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java index 57fa2c76..43db7d05 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java @@ -175,4 +175,10 @@ public void setAtomicOperation(boolean atomicOperation) { super.setAtomicOperation(atomicOperation); tableBatchOps.setAtomicOperation(atomicOperation); } + + @Override + public void setReturnOneResult(boolean returnOneResult) { + super.setReturnOneResult(returnOneResult); + tableBatchOps.setReturnOneResult(returnOneResult); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java index 4055f140..ddda9e47 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java @@ -37,6 +37,7 @@ public class BatchOperation { boolean withResult; private List operations; boolean isAtomic = false; + boolean returnOneResult = false; boolean hasCheckAndInsUp = false; /* @@ -113,6 +114,11 @@ public BatchOperation setIsAtomic(boolean isAtomic) { return this; } + public BatchOperation setReturnOneResult(boolean returnOneResult) { + this.returnOneResult = returnOneResult; + return this; + } + @SuppressWarnings("unchecked") public BatchOperationResult execute() throws Exception { if (hasCheckAndInsUp) { @@ -190,6 +196,7 @@ public BatchOperationResult executeWithNormalBatchOp() throws Exception { } } batchOps.setAtomicOperation(isAtomic); + batchOps.setReturnOneResult(returnOneResult); return new BatchOperationResult(batchOps.executeWithResult()); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java index 5ccac8f6..7773c7a5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java @@ -149,7 +149,9 @@ public void addTableOperation(ObTableOperation tableOperation) { if (isSamePropertiesNames && length > 1) { ObTableOperation prev = tableOperations.get(length - 2); ObTableOperation curr = tableOperations.get(length - 1); - if (prev.getEntity().getPropertiesCount() != curr.getEntity().getPropertiesCount()) { + if (prev.getEntity() == null || curr.getEntity() == null) { + isSamePropertiesNames = false; + } else if (prev.getEntity().getPropertiesCount() != curr.getEntity().getPropertiesCount()) { isSamePropertiesNames = false; } else { isSamePropertiesNames = diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java index eddc8881..314160ed 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java @@ -161,4 +161,10 @@ public boolean isBatchOperationAsAtomic() { public void setBatchOperationAsAtomic(boolean batchOperationAsAtomic) { this.batchOperationAsAtomic = batchOperationAsAtomic; } + + public void setBatchOpReturnOneResult(boolean returnOneResult) { + if (returnOneResult == true) { + this.option_flag.setReturnOneResult(true); + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java index 18079de0..7c191584 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java @@ -22,7 +22,7 @@ public enum ObTableOptionFlag { - DEFAULT(0), RETURNING_ROWKEY(1 << 0), USE_PUT(1 << 1); + DEFAULT(0), RETURNING_ROWKEY(1 << 0), USE_PUT(1 << 1), RETURN_ONE_RES(1 << 2); private int value; private static Map map = new HashMap(); @@ -89,4 +89,14 @@ public void setUsePut(boolean usePut) { this.value |= USE_PUT.value; } } + + public boolean isReturnOneResult() { + return (this.value & RETURN_ONE_RES.value) == 1; + } + + public void setReturnOneResult(boolean returnOneResult) { + if (returnOneResult) { + this.value |= RETURN_ONE_RES.value; + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java index 1c1c791f..46df74e4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java @@ -25,6 +25,7 @@ public abstract class AbstractTableBatchOps implements TableBatchOps { protected String tableName; protected boolean atomicOperation; + protected boolean returnOneResult; protected ObTableEntityType entityType = ObTableEntityType.DYNAMIC; @@ -131,6 +132,19 @@ public boolean isAtomicOperation() { return atomicOperation; } + @Override + public void setReturnOneResult(boolean returnOneResult) { + this.returnOneResult = returnOneResult; + } + + /* + * Is Return One Result. + */ + @Override + public boolean isReturnOneResult() { + return returnOneResult; + } + /* * Set entity type. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java index 75432b6a..7e984422 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java @@ -140,6 +140,7 @@ public void addObTableOperation(ObTableOperationType type, Object[] rowkeys, Str public List execute() throws RemotingException, InterruptedException { request.setBatchOperationAsAtomic(isAtomicOperation()); + request.setBatchOpReturnOneResult(isReturnOneResult()); Object result = obTable.execute(request); checkObTableOperationResult(result); @@ -173,6 +174,7 @@ public List execute() throws RemotingException, InterruptedException { public List executeWithResult() throws Exception { request.setBatchOperationAsAtomic(isAtomicOperation()); + request.setBatchOpReturnOneResult(isReturnOneResult()); Object result = obTable.execute(request); checkObTableOperationResult(result); diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index 5f24d0f3..bcbd5606 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -300,6 +300,7 @@ public void partitionExecute(ObTableOperationResult[] results, .toObTableConsistencyLevel()); } subRequest.setBatchOperationAsAtomic(isAtomicOperation()); + subRequest.setBatchOpReturnOneResult(isReturnOneResult()); ObTableBatchOperationResult subObTableBatchOperationResult; boolean needRefreshTableEntry = false; diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java index 0523020c..631aa8f9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java @@ -32,6 +32,10 @@ public interface TableBatchOps { boolean isAtomicOperation(); + void setReturnOneResult(boolean returnOneResult); + + boolean isReturnOneResult(); + void setEntityType(ObTableEntityType entityType); ObTableEntityType getEntityType(); diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index 758c174c..a624d5e2 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -63,6 +63,10 @@ public void teardown() throws Exception { obTableClient.delete("test_varchar_table", key); } obTableClient.delete("test_varchar_table", successKey); + for (int i = 1; i <= 9; i++) { + String key = "abcd-" + i; + obTableClient.delete("test_varchar_table", key); + } obTableClient.close(); } @@ -190,4 +194,56 @@ public void testBatchOperation() { } + @Test + public void testReturnOneRes() { + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + // default: no ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.insert("abcd-1", new String[] { "c2" }, new String[] { "returnOne-1" }); + batchOps.insert("abcd-2", new String[] { "c2" }, new String[] { "returnOne-2" }); + batchOps.insert("abcd-3", new String[] { "c2" }, new String[] { "returnOne-3" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.get(0), 1L); + Assert.assertEquals(results.get(1), 1L); + Assert.assertEquals(results.get(2), 1L); + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } + + // no atomic ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(false); + batchOps.setReturnOneResult(true); + batchOps.insert("abcd-7", new String[] { "c2" }, new String[] { "returnOne-7" }); + batchOps.insert("abcd-8", new String[] { "c2" }, new String[] { "returnOne-8" }); + batchOps.insert("abcd-9", new String[] { "c2" }, new String[] { "returnOne-9" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.get(0), 3L); + Assert.assertEquals(results.get(1), 3L); + Assert.assertEquals(results.get(2), 3L); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + + // atomic ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(true); + batchOps.setReturnOneResult(true); + batchOps.insert("abcd-4", new String[] { "c2" }, new String[] { "returnOne-4" }); + batchOps.insert("abcd-5", new String[] { "c2" }, new String[] { "returnOne-5" }); + batchOps.insert("abcd-6", new String[] { "c2" }, new String[] { "returnOne-6" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.get(0), 3L); + Assert.assertEquals(results.get(1), 3L); + Assert.assertEquals(results.get(2), 3L); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + } } From 478f98ef127f962462d53186eee64595b3cedc8c Mon Sep 17 00:00:00 2001 From: eemjwu <34029771+eemjwu@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:09:56 +0800 Subject: [PATCH 04/11] fix test case, in return one result (#97) * add option flag, batch operations can return one res * fix return one result test --- .../execute/ObTableBatchOperationRequest.java | 2 ++ .../impl/execute/ObTableOptionFlag.java | 2 ++ .../rpc/ObAtomicBatchOperationTest.java | 32 ++++++++++--------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java index 314160ed..25eeca53 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java @@ -165,6 +165,8 @@ public void setBatchOperationAsAtomic(boolean batchOperationAsAtomic) { public void setBatchOpReturnOneResult(boolean returnOneResult) { if (returnOneResult == true) { this.option_flag.setReturnOneResult(true); + } else { + this.option_flag.setReturnOneResult(false); } } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java index 7c191584..0b02e4cc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java @@ -97,6 +97,8 @@ public boolean isReturnOneResult() { public void setReturnOneResult(boolean returnOneResult) { if (returnOneResult) { this.value |= RETURN_ONE_RES.value; + } else { + this.value &= ~(RETURN_ONE_RES.value); } } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index a624d5e2..d38e9629 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -197,21 +197,6 @@ public void testBatchOperation() { @Test public void testReturnOneRes() { TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); - // default: no ReturnOneRes batch operation - try { - batchOps.clear(); - batchOps.insert("abcd-1", new String[] { "c2" }, new String[] { "returnOne-1" }); - batchOps.insert("abcd-2", new String[] { "c2" }, new String[] { "returnOne-2" }); - batchOps.insert("abcd-3", new String[] { "c2" }, new String[] { "returnOne-3" }); - List results = batchOps.execute(); - Assert.assertEquals(results.size(), 3); - Assert.assertEquals(results.get(0), 1L); - Assert.assertEquals(results.get(1), 1L); - Assert.assertEquals(results.get(2), 1L); - } catch (Exception ex) { - Assert.fail("hit exception:" + ex); - } - // no atomic ReturnOneRes batch operation try { batchOps.clear(); @@ -245,5 +230,22 @@ public void testReturnOneRes() { } catch (Exception ex) { Assert.assertTrue(ex instanceof ObTableException); } + + // default: no ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(false); + batchOps.setReturnOneResult(false); + batchOps.insert("abcd-1", new String[] { "c2" }, new String[] { "returnOne-1" }); + batchOps.insert("abcd-2", new String[] { "c2" }, new String[] { "returnOne-2" }); + batchOps.insert("abcd-3", new String[] { "c2" }, new String[] { "returnOne-3" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.get(0), 1L); + Assert.assertEquals(results.get(1), 1L); + Assert.assertEquals(results.get(2), 1L); + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } } } From cd88f21f2445741354f03f34117e0740a1f530c8 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Thu, 29 Feb 2024 21:19:21 +0800 Subject: [PATCH 05/11] add tests (#103) --- .../alipay/oceanbase/rpc/ObTablePutTest.java | 115 +++++++++++++++--- src/test/resources/ci.sql | 17 ++- 2 files changed, 115 insertions(+), 17 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java index 50f4f41c..308c426d 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java @@ -17,55 +17,75 @@ package com.alipay.oceanbase.rpc; -import com.alipay.oceanbase.rpc.mutation.Row; -import com.alipay.oceanbase.rpc.mutation.result.MutationResult; -import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.sql.Connection; +import java.sql.Statement; import java.sql.Timestamp; -import java.util.Map; import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +/** + CREATE TABLE IF NOT EXISTS `test_put` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + `c2` bigint DEFAULT NULL, + `c3` varchar(32) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + **/ public class ObTablePutTest { ObTableClient client; public static String tableName = "test_put"; @Before public void setup() throws Exception { + setMinimalImage(); final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); obTableClient.init(); this.client = obTableClient; client.addRowKeyElement(tableName, new String[] { "id" }); } - /** - CREATE TABLE IF NOT EXISTS `test_put` ( - `id` varchar(20) NOT NULL, - `c1` bigint DEFAULT NULL, - `c2` bigint DEFAULT NULL, - `c3` varchar(32) DEFAULT NULL, - `c4` bigint DEFAULT NULL, - PRIMARY KEY(`id`)) PARTITION BY KEY(`id`) PARTITIONS 32; - **/ + private static void setMinimalImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=MINIMAL"); + } + + private static void setFullImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=Full"); + } + @Test - public void testPut() throws Exception { + public void testPut1() throws Exception { try { - client.put(tableName).setRowKey(colVal("id", "id0")).execute(); + Timestamp curTs = new Timestamp(System.currentTimeMillis()); + curTs.setNanos(0); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("expired_ts", curTs)) + .execute(); client.put(tableName).setRowKey(colVal("id", "id0")) .addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("expired_ts", curTs)) .execute(); client.put(tableName).setRowKey(colVal("id", "id0")) .addMutateColVal(colVal("c1", 1L)) .addMutateColVal(colVal("c2", 2L)) + .addMutateColVal(colVal("expired_ts", curTs)) .execute(); client.put(tableName).setRowKey(colVal("id", "id0")) .addMutateColVal(colVal("c3", "c3")) + .addMutateColVal(colVal("expired_ts", curTs)) .execute(); } catch (Exception e) { e.printStackTrace(); @@ -73,7 +93,70 @@ public void testPut() throws Exception { } finally { client.delete(tableName).setRowKey(colVal("id", "id0")).execute(); } + } + + @Test + public void testPut2() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.put(tableName).setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4227][OB_ERR_NO_DEFAULT_FOR_FIELD][Field 'expired_ts' doesn't have a default value]")); + } + + @Test + public void testPut3() throws Exception { + setFullImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put(tableName).setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][binlog_row_image is full use put not supported]")); + } + @Test + public void testPut4() throws Exception { + setMinimalImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put("test_put_with_local_index").setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][table with index use put not supported]")); } + @Test + public void testPut5() throws Exception { + setMinimalImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put("test_put_with_global_index").setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][table with index use put not supported]")); + } } diff --git a/src/test/resources/ci.sql b/src/test/resources/ci.sql index 44988276..58a26773 100644 --- a/src/test/resources/ci.sql +++ b/src/test/resources/ci.sql @@ -452,6 +452,21 @@ CREATE TABLE IF NOT EXISTS `test_put` ( `c2` bigint DEFAULT NULL, `c3` varchar(32) DEFAULT NULL, `c4` bigint DEFAULT NULL, - PRIMARY KEY(`id`)) PARTITION BY KEY(`id`) PARTITIONS 32; + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + +CREATE TABLE IF NOT EXISTS `test_put_with_local_index` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + index idx1(`c1`) local, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + +CREATE TABLE IF NOT EXISTS `test_put_with_global_index` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + index idx1(`c1`) global, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; alter system set kv_hotkey_throttle_threshold = 50; From ef4946ccba011131b9dfc84fdabc2a46183dd1b4 Mon Sep 17 00:00:00 2001 From: xiaoai Date: Fri, 8 Mar 2024 15:46:56 +0800 Subject: [PATCH 06/11] support batch put (#107) --- .../com/alipay/oceanbase/rpc/mutation/BatchOperation.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java index ddda9e47..461c4e23 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java @@ -184,6 +184,11 @@ public BatchOperationResult executeWithNormalBatchOp() throws Exception { batchOps.append(mutation.getRowKey(), ((Append) mutation).getColumns(), ((Append) mutation).getValues(), withResult); break; + case PUT: + ((Put) mutation).removeRowkeyFromMutateColval(); + batchOps.append(mutation.getRowKey(), ((Put) mutation).getColumns(), + ((Put) mutation).getValues(), withResult); + break; default: throw new ObTableException("unknown operation type " + type); } From ccb3183fbe2daedfbe1202d38a872965c59eaf29 Mon Sep 17 00:00:00 2001 From: xiaoai Date: Fri, 8 Mar 2024 16:22:04 +0800 Subject: [PATCH 07/11] fix batch put use correct api (#108) * support batch put * fix batch put use put method --- .../com/alipay/oceanbase/rpc/mutation/BatchOperation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java index 461c4e23..aa4eae1e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java @@ -186,8 +186,8 @@ public BatchOperationResult executeWithNormalBatchOp() throws Exception { break; case PUT: ((Put) mutation).removeRowkeyFromMutateColval(); - batchOps.append(mutation.getRowKey(), ((Put) mutation).getColumns(), - ((Put) mutation).getValues(), withResult); + batchOps.put(mutation.getRowKey(), ((Put) mutation).getColumns(), + ((Put) mutation).getValues()); break; default: throw new ObTableException("unknown operation type " + type); From 29bfcfe7ee3392d4632096ce4f29990755e2ca01 Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Fri, 8 Mar 2024 17:46:53 +0800 Subject: [PATCH 08/11] atomic can across partition --- .../oceanbase/rpc/table/ObTableClientBatchOpsImpl.java | 6 ------ .../oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java | 6 ------ 2 files changed, 12 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index bcbd5606..a0648fb0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -256,12 +256,6 @@ public Map>>> obTableOperations.getRight().add(new ObPair(i, operation)); } - if (atomicOperation) { - if (partitionOperationsMap.size() > 1) { - throw new ObTablePartitionConsistentException( - "require atomic operation but found across partition may cause consistent problem "); - } - } return partitionOperationsMap; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index ea55b77e..f36a6393 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -253,12 +253,6 @@ public Map>>> p obTableOperations.getRight().add(new ObPair(i, operation)); } - if (atomicOperation) { - if (partitionOperationsMap.size() > 1) { - throw new ObTablePartitionConsistentException( - "require atomic operation but found across partition may cause consistent problem "); - } - } return partitionOperationsMap; } From d3367c40b4af8fd42a7cb4f9a77db090e76d6cd6 Mon Sep 17 00:00:00 2001 From: Shen Yunlong <44497386+shenyunlong@users.noreply.github.com> Date: Fri, 8 Mar 2024 18:49:49 +0800 Subject: [PATCH 09/11] [Feat] batchOperation support LSOp (#105) * [Feat] support LS batch routing * [Feat] single operation adapt to ObTableLSOp * [Test] add test for LS batch * [Enhancement] add get operation test for LsOp and fix bugs * [Test] add test cases for ObTableLSOp * [Test] add batch put test for ObTableLSOp * [Fix] add defensive code to disallow mix checkAndInsUp and other types operation * [OBKV] ObTableLsOp adapt to returnOneResult * [Fix] modify accord to review --- .../com/alipay/oceanbase/rpc/ObGlobal.java | 13 + .../alipay/oceanbase/rpc/ObTableClient.java | 61 +- .../rpc/exception/ExceptionUtil.java | 11 +- .../oceanbase/rpc/location/LocationUtil.java | 15 +- .../model/partition/ObPartitionEntry.java | 13 + .../model/partition/ObPartitionInfo.java | 1 + .../rpc/mutation/BatchOperation.java | 69 +- .../alipay/oceanbase/rpc/mutation/Put.java | 12 +- .../rpc/protocol/payload/Constants.java | 2 + .../rpc/protocol/payload/ResultCodes.java | 2 +- .../rpc/protocol/payload/impl/ObObj.java | 5 + .../protocol/payload/impl/ObTableObjType.java | 10 +- .../payload/impl/ObTableSerialUtil.java | 6 +- .../impl/execute/ObTableBatchOperation.java | 25 +- .../payload/impl/execute/ObTableLSOpFlag.java | 9 + .../impl/execute/ObTableLSOperation.java | 9 + .../impl/execute/ObTableOperationType.java | 4 +- .../payload/impl/execute/ObTableSingleOp.java | 15 + .../impl/execute/ObTableSingleOpEntity.java | 42 +- .../payload/impl/execute/ObTableTabletOp.java | 6 + .../impl/execute/ObTableTabletOpFlag.java | 13 + .../ObTableClientQueryStreamResult.java | 4 +- .../ObTableClientQueryAsyncStreamResult.java | 4 +- .../table/ObTableClientLSBatchOpsImpl.java | 376 ++++++--- .../oceanbase/rpc/table/ObTableParam.java | 17 + .../oceanbase/rpc/util/MonitorUtil.java | 16 +- .../rpc/ObAtomicBatchOperationTest.java | 59 +- .../oceanbase/rpc/ObTableClientTest.java | 4 +- .../oceanbase/rpc/ObTableLsBatchTest.java | 722 ++++++++++++++++++ .../alipay/oceanbase/rpc/ObTablePutTest.java | 24 +- 30 files changed, 1325 insertions(+), 244 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java b/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java index 01d51eaf..aab9eb62 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java @@ -81,7 +81,20 @@ public static String getObVsnString(long version) { getObVsnMajorPatch(version), getObVsnMinorPatch(version)); } + public static boolean isLsOpSupport() { + boolean isSupp = false; + if (OB_VERSION != 0) { + if (obVsnMajor() == 4 && obVsnMinor() == 2 && obVsnMajorPatch() == 3 + && OB_VERSION >= OB_VERSION_4_2_3_0) { + isSupp = true; + } + } + return isSupp; + } + public static final long OB_VERSION_4_2_1_0 = calcVersion(4, (short) 2, (byte) 1, (byte) 0); + public static final long OB_VERSION_4_2_3_0 = calcVersion(4, (short) 2, (byte) 3, (byte) 0); + public static long OB_VERSION = calcVersion(0, (short) 0, (byte) 0, (byte) 0); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index 16c5b810..0ebe6a6c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -935,7 +935,8 @@ public void syncRefreshMetadata() throws Exception { * @return the real table name */ public String getIndexTableName(final String dataTableName, final String indexName, - List scanRangeColumns, boolean forceRefreshIndexInfo) throws Exception { + List scanRangeColumns, boolean forceRefreshIndexInfo) + throws Exception { String indexTableName = dataTableName; if (indexName != null && !indexName.isEmpty() && !indexName.equalsIgnoreCase("PRIMARY")) { String tmpTableName = constructIndexTableName(dataTableName, indexName); @@ -950,7 +951,8 @@ public String getIndexTableName(final String dataTableName, final String indexNa indexTableName = tmpTableName; if (scanRangeColumns.isEmpty()) { throw new ObTableException( - "query by global index need add all index keys in order, indexTableName:" + indexTableName); + "query by global index need add all index keys in order, indexTableName:" + + indexTableName); } else { addRowKeyElement(indexTableName, scanRangeColumns.toArray(new String[scanRangeColumns.size()])); @@ -982,7 +984,8 @@ public String constructIndexTableName(final String dataTableName, final String i return "__idx_" + dataTableId + "_" + indexName; } - public ObIndexInfo getOrRefreshIndexInfo(final String indexTableName, boolean forceRefresh) throws Exception { + public ObIndexInfo getOrRefreshIndexInfo(final String indexTableName, boolean forceRefresh) + throws Exception { ObIndexInfo indexInfo = indexinfos.get(indexTableName); if (!forceRefresh && indexInfo != null) { @@ -1004,7 +1007,8 @@ public ObIndexInfo getOrRefreshIndexInfo(final String indexTableName, boolean fo if (!forceRefresh && indexInfo != null) { return indexInfo; } else { - logger.info("index info is not exist, create new index info, indexTableName: {}", indexTableName); + logger.info("index info is not exist, create new index info, indexTableName: {}", + indexTableName); int serverSize = serverRoster.getMembers().size(); int refreshTryTimes = tableEntryRefreshTryTimes > serverSize ? serverSize : tableEntryRefreshTryTimes; @@ -1533,6 +1537,7 @@ public ObPair getTable(String tableName, TableEntry tableEnt long partIdx = tableEntry.getPartIdx(partId); partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() .getPartTabletIdMap().get(partIdx) : partId; + param.setLsId(tableEntry.getPartitionEntry().getLsId(partId)); } param.setTableId(tableEntry.getTableId()); @@ -2013,30 +2018,30 @@ public ObPayload putWithResult(final String tableName, final Object[] rowKey, final Object[] values) throws Exception { final long start = System.currentTimeMillis(); return executeMutation(tableName, - new MutationExecuteCallback(rowKey, keyRanges) { - /** - * Execute. - */ - @Override - public ObPayload execute(ObPair obPair) throws Exception { - long TableTime = System.currentTimeMillis(); - ObTableParam tableParam = obPair.getRight(); - ObTable obTable = tableParam.getObTable(); - ObTableOperationRequest request = ObTableOperationRequest.getInstance( - tableName, PUT, rowKey, columns, values, - obTable.getObTableOperationTimeout()); - request.setTableId(tableParam.getTableId()); - // partId/tabletId - request.setPartitionId(tableParam.getPartitionId()); - ObPayload result = obTable.execute(request); - String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "PUT", endpoint, rowKey, - (ObTableOperationResult) result, TableTime - start, - System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); - checkResult(obTable.getIp(), obTable.getPort(), request, result); - return result; - } - }); + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, PUT, rowKey, columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "PUT", endpoint, rowKey, + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); } /** diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java index 152f4bc3..73a9ad93 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java @@ -72,14 +72,15 @@ public static ObTableException convertToObTableException(String host, int port, } if (resultCodes.errorCode == OB_ERR_KV_GLOBAL_INDEX_ROUTE.errorCode) { - return new ObTableGlobalIndexRouteException("[" + String.valueOf(resultCodes.errorCode) + "]" + "[" - + resultCodes.name() + "]" + "[" + errMsg + "]" + "[" + server - + "]" + "[" + trace + "]", resultCodes.errorCode); + return new ObTableGlobalIndexRouteException("[" + String.valueOf(resultCodes.errorCode) + + "]" + "[" + resultCodes.name() + "]" + + "[" + errMsg + "]" + "[" + server + "]" + + "[" + trace + "]", resultCodes.errorCode); } else { // [errCode][errCodeName][errMsg][server][trace] return new ObTableException("[" + String.valueOf(resultCodes.errorCode) + "]" + "[" - + resultCodes.name() + "]" + "[" + errMsg + "]" + "[" + server - + "]" + "[" + trace + "]", resultCodes.errorCode); + + resultCodes.name() + "]" + "[" + errMsg + "]" + "[" + + server + "]" + "[" + trace + "]", resultCodes.errorCode); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index e95f3e16..00f97e3f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -49,6 +49,7 @@ import static com.alipay.oceanbase.rpc.util.RandomUtil.getRandomNum; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.*; import static java.lang.String.format; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_LS_ID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -154,9 +155,11 @@ public class LocationUtil { private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " + + ", A.spare1 as replica_type, D.ls_id as ls_id " + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and tablet_id in ({0})"; + + "inner join oceanbase.DBA_OB_TENANTS C on C.tenant_name = A.tenant_name " + + "left join oceanbase.CDB_OB_TABLET_TO_LS D on D.tenant_id = C.tenant_id and D.tablet_id = A.tablet_id " + + "WHERE C.tenant_name = ? and database_name= ? and table_name = ? and A.tablet_id in ({0}) "; private static final String PROXY_FIRST_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, tablet_id, high_bound_val, sub_part_num " + "FROM oceanbase.__all_virtual_proxy_partition " @@ -945,11 +948,18 @@ private static ObPartitionEntry getPartitionLocationFromResultSet(TableEntry tab throws SQLException, ObTablePartitionLocationRefreshException { Map partitionLocation = new HashMap(); + Map tabletLsIdMap = new HashMap<>(); while (rs.next()) { ReplicaLocation replica = buildReplicaLocation(rs); long partitionId; if (ObGlobal.obVsnMajor() >= 4) { partitionId = rs.getLong("tablet_id"); + long lsId = rs.getLong("ls_id"); + if (!rs.wasNull()) { + tabletLsIdMap.put(partitionId, lsId); + } else { + tabletLsIdMap.put(partitionId, INVALID_LS_ID); // non-partitioned table + } } else { partitionId = rs.getLong("partition_id"); if (tableEntry.isPartitionTable() @@ -975,6 +985,7 @@ private static ObPartitionEntry getPartitionLocationFromResultSet(TableEntry tab } ObPartitionEntry partitionEntry = new ObPartitionEntry(); partitionEntry.setPartitionLocation(partitionLocation); + partitionEntry.setTabletLsIdMap(tabletLsIdMap); if (ObGlobal.obVsnMajor() < 4) { for (long i = 0; i < tableEntry.getPartitionNum(); i++) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java index 8d63a16b..d3e87e71 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java @@ -25,6 +25,9 @@ public class ObPartitionEntry { private Map partitionLocation = new HashMap(); + // mapping from tablet id to ls id, and the part id to tablet id mapping is in ObPartitionInfo + private Map tabletLsIdMap = new HashMap<>(); + public Map getPartitionLocation() { return partitionLocation; } @@ -83,4 +86,14 @@ public void prepareForWeakRead(ObServerLdcLocation ldcLocation) { public String toString() { return "ObPartitionEntry{" + "partitionLocation=" + partitionLocation + '}'; } + + public Map getTabletLsIdMap() { + return tabletLsIdMap; + } + + public void setTabletLsIdMap(Map tabletLsIdMap) { + this.tabletLsIdMap = tabletLsIdMap; + } + + public long getLsId(long tabletId) { return tabletLsIdMap.get(tabletId); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java index de840247..b6d9e083 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java @@ -30,6 +30,7 @@ public class ObPartitionInfo { private ObPartDesc firstPartDesc = null; private ObPartDesc subPartDesc = null; private List partColumns = new ArrayList(1); + // mapping from part id to tablet id, and the tablet id to ls id mapping is in ObPartitionInfo private Map partTabletIdMap = null; private Map partNameIdMap = null; private Map rowKeyElement = null; diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java index aa4eae1e..4ba9a7b0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java @@ -26,6 +26,7 @@ import com.alipay.oceanbase.rpc.table.api.Table; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.ObGlobal; import java.util.ArrayList; import java.util.Arrays; @@ -39,6 +40,9 @@ public class BatchOperation { boolean isAtomic = false; boolean returnOneResult = false; boolean hasCheckAndInsUp = false; + boolean hasGet = false; + ObTableOperationType lastType = ObTableOperationType.INVALID; + boolean isSameType = true; /* * default constructor @@ -80,6 +84,12 @@ public BatchOperation setTable(String tableName) { * add queries */ public BatchOperation addOperation(TableQuery... queries) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != ObTableOperationType.GET) { + isSameType = false; + } + + lastType = ObTableOperationType.GET; this.operations.addAll(Arrays.asList(queries)); return this; } @@ -88,6 +98,13 @@ public BatchOperation addOperation(TableQuery... queries) { * add mutations */ public BatchOperation addOperation(Mutation... mutations) { + for (int i = 0; i < mutations.length; i++) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != mutations[i].getOperationType()) { + isSameType = false; + } + lastType = mutations[i].getOperationType(); + } this.operations.addAll(Arrays.asList(mutations)); return this; } @@ -96,6 +113,13 @@ public BatchOperation addOperation(Mutation... mutations) { * add mutations */ public BatchOperation addOperation(List mutations) { + for (int i = 0; i < mutations.size(); i++) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != mutations.get(i).getOperationType()) { + isSameType = false; + } + lastType = mutations.get(i).getOperationType(); + } this.operations.addAll(mutations); return this; } @@ -104,6 +128,11 @@ public BatchOperation addOperation(List mutations) { * add CheckAndInsUp */ public BatchOperation addOperation(CheckAndInsUp... insUps) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != ObTableOperationType.CHECK_AND_INSERT_UP) { + isSameType = false; + } + lastType = ObTableOperationType.CHECK_AND_INSERT_UP; this.operations.addAll(Arrays.asList(insUps)); this.hasCheckAndInsUp = true; return this; @@ -121,14 +150,22 @@ public BatchOperation setReturnOneResult(boolean returnOneResult) { @SuppressWarnings("unchecked") public BatchOperationResult execute() throws Exception { - if (hasCheckAndInsUp) { + if (returnOneResult + && !(isSameType && (lastType == ObTableOperationType.INSERT + || lastType == ObTableOperationType.PUT + || lastType == ObTableOperationType.REPLACE || lastType == ObTableOperationType.DEL))) { + throw new IllegalArgumentException( + "returnOneResult only support multi-insert/put/replace/del"); + } + + if (hasCheckAndInsUp || ObGlobal.isLsOpSupport()) { return executeWithLSBatchOp(); } else { return executeWithNormalBatchOp(); } } - public BatchOperationResult executeWithNormalBatchOp() throws Exception { + private BatchOperationResult executeWithNormalBatchOp() throws Exception { if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } @@ -205,16 +242,19 @@ public BatchOperationResult executeWithNormalBatchOp() throws Exception { return new BatchOperationResult(batchOps.executeWithResult()); } - public BatchOperationResult executeWithLSBatchOp() throws Exception { + private BatchOperationResult executeWithLSBatchOp() throws Exception { if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } ObTableClientLSBatchOpsImpl batchOps; + boolean hasSetRowkeyElement = false; + int checkAndInsUPCnt = 0; + if (client instanceof ObTableClient) { batchOps = new ObTableClientLSBatchOpsImpl(tableName, (ObTableClient) client); - boolean hasSetRowkeyElement = false; for (Object operation : operations) { if (operation instanceof CheckAndInsUp) { + checkAndInsUPCnt++; CheckAndInsUp checkAndInsUp = (CheckAndInsUp) operation; batchOps.addOperation(checkAndInsUp); List rowKeyNames = checkAndInsUp.getInsUp().getRowKeyNames(); @@ -223,6 +263,18 @@ public BatchOperationResult executeWithLSBatchOp() throws Exception { rowKeyNames.toArray(new String[0])); hasSetRowkeyElement = true; } + } else if (operation instanceof Mutation) { + Mutation mutation = (Mutation) operation; + batchOps.addOperation(mutation); + if (!hasSetRowkeyElement && mutation.getRowKeyNames() != null) { + List rowKeyNames = mutation.getRowKeyNames(); + ((ObTableClient) client).addRowKeyElement(tableName, + rowKeyNames.toArray(new String[0])); + hasSetRowkeyElement = true; + } + } else if (operation instanceof TableQuery) { + TableQuery query = (TableQuery) operation; + batchOps.addOperation(query); } else { throw new IllegalArgumentException( "The operations in batch must be all checkAndInsUp or all non-checkAndInsUp"); @@ -232,6 +284,15 @@ public BatchOperationResult executeWithLSBatchOp() throws Exception { throw new IllegalArgumentException( "execute batch using ObTable diretly is not supporeted"); } + + if (checkAndInsUPCnt > 0 && checkAndInsUPCnt != operations.size()) { + throw new IllegalArgumentException( + "Can not mix checkAndInsUP and other types operation in batch"); + } + + batchOps.setReturningAffectedEntity(withResult); + batchOps.setReturnOneResult(returnOneResult); + batchOps.setAtomicOperation(isAtomic); return new BatchOperationResult(batchOps.executeWithResult()); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java index 79cf56c5..a4ca7730 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java @@ -155,17 +155,15 @@ public MutationResult execute() throws Exception { removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); if (null == getQuery()) { // simple Put, without filter - return new MutationResult(((ObTableClient) getClient()).putWithResult( - getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), - values.toArray())); + return new MutationResult(((ObTableClient) getClient()).putWithResult(getTableName(), + getRowKey(), getKeyRanges(), columns.toArray(new String[0]), values.toArray())); } else { if (checkMutationWithFilter()) { // QueryAndPut - ObTableOperation operation = ObTableOperation.getInstance( - ObTableOperationType.PUT, getRowKey(), columns.toArray(new String[0]), - values.toArray()); + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.PUT, + getRowKey(), columns.toArray(new String[0]), values.toArray()); return new MutationResult(((ObTableClient) getClient()).mutationWithFilter( - getQuery(), getRowKey(), getKeyRanges(), operation, true)); + getQuery(), getRowKey(), getKeyRanges(), operation, true)); } else { throw new ObTableUnexpectedException("should set filter and scan range both"); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java index 065d02d7..26ecc1cc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java @@ -31,4 +31,6 @@ public interface Constants { long UNSIGNED_INT32_MAX = (1L << 32) - 1; + long INVALID_LS_ID = -1; + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java index 3eb331a8..45eb61c5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java @@ -723,7 +723,7 @@ public enum ResultCodes { OB_CLUSTER_NAME_NOT_EQUAL(-9016), // OB_RS_LIST_INVAILD(-9017), // OB_AGENT_HAS_FAILED_TASK(-9018), // - OB_ERR_KV_GLOBAL_INDEX_ROUTE(-10500),// + OB_ERR_KV_GLOBAL_INDEX_ROUTE(-10500), // OB_KV_CREDENTIAL_NOT_MATCH(-10509), // OB_KV_ROWKEY_COUNT_NOT_MATCH(-10510), // OB_KV_COLUMN_TYPE_NOT_MATCH(-10511), // diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java index 7cc115a7..7fa4e1b5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java @@ -198,4 +198,9 @@ public boolean isMaxObj() { return (getMeta().getType() == ObObjType.ObExtendType) && ((long) getValue() == MAX_OBJECT_VALUE); } + + // set value directly, do not wrapped by ObVString when type is varchar + public void setValueFromTableObj(Object value) { + this.value = value; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java index 2428db4b..62765a85 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java @@ -50,7 +50,7 @@ public void decode(ByteBuf buf, ObObj obj) { ObObjMeta objMeta = objType.getDefaultObjMeta(); objMeta.setCsType(ObCollationType.CS_TYPE_BINARY); obj.setMeta(objMeta); - obj.setValue(objType.decode(buf, objMeta.getCsType())); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); } }, @@ -107,7 +107,7 @@ public void decode(ByteBuf buf, ObObj obj) { ObObjType objType = getObjType(this); ObObjMeta objMeta = objType.getDefaultObjMeta(); obj.setMeta(objMeta); - obj.setValue(-2L); + obj.setValueFromTableObj(-2L); } public int getEncodedSize(ObObj obj) { @@ -128,7 +128,7 @@ public void decode(ByteBuf buf, ObObj obj) { ObObjType objType = getObjType(this); ObObjMeta objMeta = objType.getDefaultObjMeta(); obj.setMeta(objMeta); - obj.setValue(-3L); + obj.setValueFromTableObj(-3L); } public int getEncodedSize(ObObj obj) { @@ -260,7 +260,7 @@ public void decode(ByteBuf buf, ObObj obj) { ObObjType objType = getObjType(this); ObObjMeta objMeta = objType.getDefaultObjMeta(); obj.setMeta(objMeta); - obj.setValue(objType.decode(buf, objMeta.getCsType())); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); } public int getEncodedSize(ObObj obj) { @@ -297,7 +297,7 @@ public void decodeWithMeta(ByteBuf buf, ObObj obj) { meta.setCsLevel(ObCollationLevel.valueOf(Serialization.decodeI8(buf.readByte()))); meta.setCsType(ObCollationType.valueOf(Serialization.decodeI8(buf.readByte()))); meta.setScale(Serialization.decodeI8(buf.readByte())); - obj.setValue(objType.decode(buf, meta.getCsType())); + obj.setValueFromTableObj(objType.decode(buf, meta.getCsType())); } public int getEncodedSizeWithMeta(ObObj obj) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java index 29805cd3..e32c5ffb 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java @@ -44,7 +44,11 @@ public static ObTableObjType decodeTableObjType(ByteBuf buf) { throw new IllegalArgumentException("cannot get ObTableObjType, buf is null"); } byte type = Serialization.decodeI8(buf); - return ObTableObjType.valueOf(type); + ObTableObjType objType = ObTableObjType.valueOf(type); + if (objType == null) { + throw new IllegalArgumentException("cannot get table object type from value"); + } + return objType; } static public int getEncodedSize(ObNewRange range) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java index 7773c7a5..84b03df6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java @@ -33,9 +33,9 @@ */ public class ObTableBatchOperation extends AbstractPayload { - private List tableOperations = new ArrayList(); - private boolean isReadOnly = true; - private boolean isSameType = true; + private List tableOperations = new ArrayList(); + private boolean isReadOnly = true; + private boolean isSameType = true; private boolean isSamePropertiesNames = true; /* @@ -140,8 +140,8 @@ public void addTableOperation(ObTableOperation tableOperation) { isReadOnly = false; } if (isSameType - && length > 1 - && tableOperations.get(length - 1).getOperationType() != tableOperations + && length > 1 + && tableOperations.get(length - 1).getOperationType() != tableOperations .get(length - 2).getOperationType()) { isSameType = false; } @@ -151,11 +151,13 @@ public void addTableOperation(ObTableOperation tableOperation) { ObTableOperation curr = tableOperations.get(length - 1); if (prev.getEntity() == null || curr.getEntity() == null) { isSamePropertiesNames = false; - } else if (prev.getEntity().getPropertiesCount() != curr.getEntity().getPropertiesCount()) { + } else if (prev.getEntity().getPropertiesCount() != curr.getEntity() + .getPropertiesCount()) { isSamePropertiesNames = false; } else { - isSamePropertiesNames = - mapKeysToSetIgnoringCase(prev.getEntity().getProperties().keySet()).equals(mapKeysToSetIgnoringCase(curr.getEntity().getProperties().keySet())); + isSamePropertiesNames = mapKeysToSetIgnoringCase( + prev.getEntity().getProperties().keySet()).equals( + mapKeysToSetIgnoringCase(curr.getEntity().getProperties().keySet())); } } } @@ -183,12 +185,13 @@ public void setTableOperations(List tableOperations) { if (this.isSamePropertiesNames) { if (firstKeySetIgnoreCase == null) { - firstKeySetIgnoreCase = mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet()); + firstKeySetIgnoreCase = mapKeysToSetIgnoringCase(o.getEntity() + .getProperties().keySet()); } else if (firstKeySetIgnoreCase.size() != o.getEntity().getPropertiesCount()) { this.isSamePropertiesNames = false; } else { - this.isSamePropertiesNames = - firstKeySetIgnoreCase.equals(mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet())); + this.isSamePropertiesNames = firstKeySetIgnoreCase + .equals(mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet())); } } } else { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java index 6c704adb..b2a56e8c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java @@ -20,6 +20,7 @@ public class ObTableLSOpFlag { private static final int FLAG_IS_SAME_TYPE = 1 << 0; private static final int FLAG_IS_SAME_PROPERTIES_NAMES = 1 << 1; + private static final int FLAG_RETURN_ONE_RESULT = 1 << 2; private long flags = 0; public void setFlagIsSameType(boolean isSameType) { @@ -38,6 +39,14 @@ public void setFlagIsSamePropertiesNames(boolean isSamePropertiesNames) { } } + public void setReturnOneResult(boolean returnOneResult) { + if (returnOneResult) { + flags |= FLAG_RETURN_ONE_RESULT; + } else { + flags &= ~FLAG_RETURN_ONE_RESULT; + } + } + public long getValue() { return flags; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java index 4405285b..4efe2b5d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java @@ -227,6 +227,10 @@ public void setLsId(long lsId) { this.lsId = lsId; } + public void setReturnOneResult(boolean returnOneResult) { + optionFlag.setReturnOneResult(returnOneResult); + } + public boolean isSameType() { return optionFlag.getFlagIsSameType(); } @@ -326,4 +330,9 @@ public void prepare() { this.prepareOption(); this.prepareColumnNamesBitMap(); } + + public long getLsId() { + return lsId; + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java index 76faf2e9..8c69844b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java @@ -28,8 +28,7 @@ public enum ObTableOperationType { APPEND(7), // append column value SCAN(8), // query TTL(9), // observer internal type, not used by client - CHECK_AND_INSERT_UP(10), - PUT(11), // override row + CHECK_AND_INSERT_UP(10), PUT(11), // override row TRIGGER(12), // internal op type INVALID(15); @@ -46,6 +45,7 @@ public enum ObTableOperationType { false, // SCAN false, // TTL true, // CHECK_AND_INSERT_UP + false, // PUT false // INVALID ); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java index ebe3a797..79271e00 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java @@ -18,12 +18,15 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; import java.util.ArrayList; import java.util.List; +import java.util.logging.Logger; public class ObTableSingleOp extends AbstractPayload { private ObTableOperationType singleOpType; @@ -152,4 +155,16 @@ public void setEntities(List entities) { public void addEntity(ObTableSingleOpEntity entity) { this.entities.add(entity); } + + public List getRowkeyObjs() { + List rowkeyObjs; + if (singleOpType == ObTableOperationType.SCAN) { + throw new IllegalArgumentException("can not get rowkey from scan operation"); + } else if (singleOpType == ObTableOperationType.CHECK_AND_INSERT_UP) { + rowkeyObjs = getScanRange().get(0).getStartKey().getObjs(); + } else { + rowkeyObjs = entities.get(0).getRowkey(); + } + return rowkeyObjs; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java index 460e0539..d9a440f4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java @@ -104,12 +104,17 @@ public byte[] encode() { private byte[] parseBitMap(long bitLen, List aggColumnNames, List columnNames, ByteBuf buf) { byte[] bitMap = new byte[(int) Math.ceil(bitLen / 8.0)]; - for (int i = 0; i < bitMap.length; i++) { - bitMap[i] = Serialization.decodeI8(buf); - for (int j = 0; j < 8; j++) { - if ((bitMap[i] & (1 << j)) != 0) { - if (i * 8 + j < aggColumnNames.size()) { - columnNames.add(aggColumnNames.get(i * 8 + j)); + if (bitLen == 0) { + // is same properties names + columnNames.addAll(aggColumnNames); + } else { + for (int i = 0; i < bitMap.length; i++) { + bitMap[i] = Serialization.decodeI8(buf); + for (int j = 0; j < 8; j++) { + if ((bitMap[i] & (1 << j)) != 0) { + if (i * 8 + j < aggColumnNames.size()) { + columnNames.add(aggColumnNames.get(i * 8 + j)); + } } } } @@ -195,12 +200,12 @@ public static boolean areArraysSameLengthOrBothNull(Object[] a, Object[] b) { public static ObTableSingleOpEntity getInstance(String[] rowKeyNames, Object[] rowKey, String[] propertiesNames, Object[] propertiesValues) { ObTableSingleOpEntity entity = new ObTableSingleOpEntity(); - if (!areArraysSameLengthOrBothNull(rowKeyNames, rowKey) || !areArraysSameLengthOrBothNull(propertiesNames, propertiesValues)) { + if (!areArraysSameLengthOrBothNull(rowKeyNames, rowKey)) { throw new IllegalArgumentException(String.format( - "column length should be equals to values length, rowkeyNames: %s, rowkey: %s," - + "propertiesNames: %s," + "propertiesValues: %s",rowKeyNames, - rowKey, propertiesNames, propertiesValues)); + "rowKey names length should be equals to rowKey values length, rowkeyNames: %s, rowkey: %s,", + rowKeyNames, rowKey)); } + if (rowKey != null) { for (int i = 0; i < rowKey.length; i++) { String name = rowKeyNames[i]; @@ -212,12 +217,14 @@ public static ObTableSingleOpEntity getInstance(String[] rowKeyNames, Object[] r if (propertiesNames != null) { for (int i = 0; i < propertiesNames.length; i++) { String name = propertiesNames[i]; - Object value = propertiesValues[i]; + Object value = null; + if (propertiesValues != null) { + value = propertiesValues[i]; + } ObObj c = ObObj.getInstance(value); entity.addPropertyValue(name, c); } } - return entity; } @@ -346,8 +353,17 @@ public void setAggPropertiesNames(List columnNames) { public Map getSimpleProperties() { Map values = new HashMap((int) propertiesValues.size()); for (int i = 0; i < propertiesValues.size(); i++) { - values.put(propertiesNames.get(i), propertiesValues.get(i)); + values.put(propertiesNames.get(i), propertiesValues.get(i).getValue()); } return values; } + + public List getRowkey() { + return rowkey; + } + + public void setRowkey(List rowkey) { + this.rowkey = rowkey; + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java index e730ab11..11d17f30 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java @@ -140,6 +140,10 @@ public void setSingleOperations(List singleOperations) { prevType = o.getSingleOpType(); } } + + if (isSameType() && singleOperations.get(0).getSingleOpType() == ObTableOperationType.GET) { + setIsReadOnly(true); + } this.singleOperations = singleOperations; } @@ -155,6 +159,8 @@ public long getTabletId() { public void setIsSameType(boolean isSameType) { optionFlag.setFlagIsSameType(isSameType);} + public void setIsReadOnly(boolean isReadOnly) { optionFlag.setFlagIsReadOnly(isReadOnly);} + public Set getRowKeyNamesSet() { return rowKeyNamesSet; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java index 32c97ad8..0e28ea59 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java @@ -21,6 +21,7 @@ public class ObTableTabletOpFlag { private static final int FLAG_IS_SAME_TYPE = 1 << 0; // Maybe useless, we use isSameProperties flag from LSOp private static final int FLAG_IS_SAME_PROPERTIES_NAMES = 1 << 1; + private static final int FLAG_IS_READ_ONLY = 1 << 2; private long flags = 0; public void setFlagIsSameType(boolean isSameType) { @@ -39,6 +40,14 @@ public void setFlagIsSamePropertiesNames(boolean isSamePropertiesNames) { } } + public void setFlagIsReadOnly(boolean isReadOnly) { + if (isReadOnly) { + flags |= FLAG_IS_READ_ONLY; + } else { + flags &= ~FLAG_IS_READ_ONLY; + } + } + public long getValue() { return flags; } @@ -54,4 +63,8 @@ public boolean getFlagIsSameType() { public boolean getFlagIsSamePropertiesNames() { return (flags & FLAG_IS_SAME_PROPERTIES_NAMES) != 0; } + + public boolean getFlagIsReadOnly() { + return (flags & FLAG_IS_READ_ONLY) != 0; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java index 17677930..f989c365 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java @@ -152,8 +152,8 @@ protected ObTableQueryResult execute(ObPair partIdWithIndex, "meet global index route expcetion: indexTableName:{} partition id:{}, errorCode: {}, retry times {}", indexTableName, partIdWithIndex.getLeft(), ((ObTableException) e).getErrorCode(), tryTimes, e); - indexTableName = client.getIndexTableName(tableName, tableQuery.getIndexName(), - tableQuery.getScanRangeColumns(), true); + indexTableName = client.getIndexTableName(tableName, + tableQuery.getIndexName(), tableQuery.getScanRangeColumns(), true); } else { logger .warn( diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/async/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/async/ObTableClientQueryAsyncStreamResult.java index b24ab87f..9635830e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/async/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/async/ObTableClientQueryAsyncStreamResult.java @@ -100,8 +100,8 @@ protected ObTableQueryAsyncResult executeAsync(ObPair partId "meet global index route expcetion: indexTableName:{} partition id:{}, errorCode: {}, retry times {}", indexTableName, partIdWithObTable.getLeft(), ((ObTableException) e).getErrorCode(), tryTimes, e); - indexTableName = client.getIndexTableName(tableName, tableQuery.getIndexName(), - tableQuery.getScanRangeColumns(), true); + indexTableName = client.getIndexTableName(tableName, + tableQuery.getIndexName(), tableQuery.getScanRangeColumns(), true); } else { logger .warn( diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index f36a6393..58637d86 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -22,19 +22,18 @@ import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObServerRoute; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; -import com.alipay.oceanbase.rpc.mutation.InsertOrUpdate; -import com.alipay.oceanbase.rpc.mutation.Mutation; +import com.alipay.oceanbase.rpc.mutation.*; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutate; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.MonitorUtil; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.slf4j.Logger; +import com.alipay.oceanbase.rpc.table.api.TableQuery; import java.util.*; import java.util.concurrent.ExecutorService; @@ -42,6 +41,7 @@ import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.RUNTIME; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.*; public class ObTableClientLSBatchOpsImpl extends AbstractTableBatchOps { @@ -177,12 +177,105 @@ public void addOperation(CheckAndInsUp checkAndInsUp) { addOperation(singleOp); } + public void addOperation(TableQuery query) throws Exception { + // entity + String[] rowKeyNames = query.getRowKey().getColumns(); + if (rowKeyNames == null || rowKeyNames.length == 0) { + throw new IllegalArgumentException("rowKey is empty in get op"); + } + Object[] rowKey = query.getRowKey().getValues(); + String[] propertiesNames = query.getSelectColumns().toArray(new String[0]); + ObTableSingleOpEntity entity = ObTableSingleOpEntity.getInstance(rowKeyNames, rowKey, + propertiesNames, null); + + ObTableSingleOp singleOp = new ObTableSingleOp(); + singleOp.setSingleOpType(ObTableOperationType.GET); + singleOp.addEntity(entity); + addOperation(singleOp); + } + + public void addOperation(Mutation mutation) throws Exception { + // entity + String[] rowKeyNames = null; + Object[] rowKey = null; + String[] propertiesNames = null; + Object[] propertiesValues = null; + + ObTableOperationType type = mutation.getOperationType(); + switch (type) { + case GET: + throw new IllegalArgumentException("Invalid type in batch operation, " + type); + case INSERT: + ((Insert) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Insert) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Insert) mutation).getColumns(); + propertiesValues = ((Insert) mutation).getValues(); + break; + case DEL: + rowKeyNames = ((Delete) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + break; + case UPDATE: + ((Update) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Update) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Update) mutation).getColumns(); + propertiesValues = ((Update) mutation).getValues(); + break; + case INSERT_OR_UPDATE: + ((InsertOrUpdate) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((InsertOrUpdate) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((InsertOrUpdate) mutation).getColumns(); + propertiesValues = ((InsertOrUpdate) mutation).getValues(); + break; + case REPLACE: + ((Replace) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Replace) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Replace) mutation).getColumns(); + propertiesValues = ((Replace) mutation).getValues(); + break; + case INCREMENT: + ((Increment) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Increment) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Increment) mutation).getColumns(); + propertiesValues = ((Increment) mutation).getValues(); + break; + case APPEND: + ((Append) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Append) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Append) mutation).getColumns(); + propertiesValues = ((Append) mutation).getValues(); + break; + case PUT: + ((Put) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Put) mutation).getRowKeyNames().toArray(new String[0]); + rowKey = mutation.getRowKey(); + propertiesNames = ((Put) mutation).getColumns(); + propertiesValues = ((Put) mutation).getValues(); + break; + default: + throw new ObTableException("unknown operation type " + type); + } + + ObTableSingleOpEntity entity = ObTableSingleOpEntity.getInstance(rowKeyNames, rowKey, + propertiesNames, propertiesValues); + ObTableSingleOp singleOp = new ObTableSingleOp(); + singleOp.setSingleOpType(type); + singleOp.addEntity(entity); + addOperation(singleOp); + } + /* * Execute. */ public List execute() throws Exception { List results = new ArrayList(batchOperation.size()); - for (ObTableSingleOpResult result : executeInternal().getResults()) { + for (ObTableSingleOpResult result : executeInternal()) { int errCode = result.getHeader().getErrno(); if (errCode == ResultCodes.OB_SUCCESS.errorCode) { results.add(result.getAffectedRows()); @@ -200,7 +293,7 @@ public List execute() throws Exception { */ public List executeWithResult() throws Exception { List results = new ArrayList(batchOperation.size()); - for (ObTableSingleOpResult result : executeInternal().getResults()) { + for (ObTableSingleOpResult result : executeInternal()) { int errCode = result.getHeader().getErrno(); if (errCode == ResultCodes.OB_SUCCESS.errorCode) { results.add(new MutationResult(result)); @@ -213,16 +306,16 @@ public List executeWithResult() throws Exception { return results; } - public Map>>> partitionPrepare() + public Map>>>> partitionPrepare() throws Exception { - // TODO: currently, we only support tablet level operation aggregation List operations = getSingleOperations(); - // map: > - Map>>> partitionOperationsMap = + // map: >>> + Map>>>> lsOperationsMap = new HashMap(); // In ODP mode, client send the request to ODP directly without route if (obTableClient.isOdpMode()) { + Map>>> tabletOperationsMap = new HashMap<>(); ObPair>> obTableOperations = new ObPair(new ObTableParam(obTableClient.getOdpTable()), new ArrayList>()); @@ -230,65 +323,103 @@ public Map>>> p ObTableSingleOp operation = operations.get(i); obTableOperations.getRight().add(new ObPair(i, operation)); } - partitionOperationsMap.put(0L, obTableOperations); - return partitionOperationsMap; + tabletOperationsMap.put(INVALID_TABLET_ID, obTableOperations); + lsOperationsMap.put(INVALID_LS_ID, tabletOperationsMap); + return lsOperationsMap; } for (int i = 0; i < operations.size(); i++) { ObTableSingleOp operation = operations.get(i); - ObRowKey rowKeyObject = operation.getScanRange().get(0).getStartKey(); - int rowKeySize = rowKeyObject.getObjs().size(); + List rowkeyObjs = operation.getRowkeyObjs(); + int rowKeySize = rowkeyObjs.size(); Object[] rowKey = new Object[rowKeySize]; for (int j = 0; j < rowKeySize; j++) { - rowKey[j] = rowKeyObject.getObj(j).getValue(); + rowKey[j] = rowkeyObjs.get(j).getValue(); } ObPair tableObPair= obTableClient.getTable(tableName, rowKey, false, false, obTableClient.getRoute(false)); - ObPair>> obTableOperations = - partitionOperationsMap.get(tableObPair.getLeft()); - if (obTableOperations == null) { - obTableOperations = new ObPair<>(tableObPair.getRight(), new ArrayList<>()); - partitionOperationsMap.put(tableObPair.getLeft(), obTableOperations); + long lsId = tableObPair.getRight().getLsId(); + + Map>>> tabletOperations + = lsOperationsMap.get(lsId); + // if ls id not exists + if (tabletOperations == null) { + tabletOperations = new HashMap<>(); + lsOperationsMap.put(lsId, tabletOperations); } - obTableOperations.getRight().add(new ObPair(i, operation)); + + ObPair>> singleOperations = + tabletOperations.get(tableObPair.getLeft()); + // if tablet id not exists + if (singleOperations == null) { + singleOperations = new ObPair<>(tableObPair.getRight(), new ArrayList<>()); + tabletOperations.put(tableObPair.getLeft(), singleOperations); + } + + singleOperations.getRight().add(new ObPair(i, operation)); } - return partitionOperationsMap; + return lsOperationsMap; } /* * Partition execute. */ public void partitionExecute(ObTableSingleOpResult[] results, - Map.Entry>>> partitionOperation) + Map.Entry>>>> lsOperation) throws Exception { - ObTableParam tableParam = partitionOperation.getValue().getLeft(); - long tableId = tableParam.getTableId(); - long partId = tableParam.getPartitionId(); - long originPartId = tableParam.getPartId(); - ObTable subObTable = tableParam.getObTable(); - List> subOperationWithIndexList = partitionOperation - .getValue().getRight(); - - ObTableLSOpRequest subRequest = new ObTableLSOpRequest(); - List subOperations = new ArrayList<>(); - for (ObPair operationWithIndex : subOperationWithIndexList) { - subOperations.add(operationWithIndex.getRight()); + long lsId = lsOperation.getKey(); + Map>>> tabletOperationsMap = lsOperation.getValue(); + if (tabletOperationsMap.size() == 0) { + logger.warn("the size of tablet operations in ls operation is zero"); + throw new ObTableUnexpectedException("the size of tablet operations in ls operation is zero"); + } + + ObTableLSOpRequest tableLsOpRequest = new ObTableLSOpRequest(); + ObTableLSOperation tableLsOp = new ObTableLSOperation(); + tableLsOp.setLsId(lsId); + tableLsOp.setReturnOneResult(returnOneResult); + // fetch the following parameters in first entry for routing + long tableId = 0; + long originPartId = 0; + long operationTimeout = 0; + ObTable subObTable = null; + + boolean isFirstEntry = true; + // list ( index list for tablet op 1, index list for tablet op 2, ...) + List>> lsOperationWithIndexList = new ArrayList<>(); + for (final Map.Entry>>> tabletOperation : tabletOperationsMap.entrySet()) { + ObTableParam tableParam = tabletOperation.getValue().getLeft(); + long tabletId = tableParam.getPartitionId(); + List> tabletOperationWithIndexList = tabletOperation.getValue().getRight(); + lsOperationWithIndexList.add(tabletOperationWithIndexList); + List singleOps = new ArrayList<>(); + for (ObPair operationWithIndex : tabletOperationWithIndexList) { + singleOps.add(operationWithIndex.getRight()); + } + ObTableTabletOp tableTabletOp = new ObTableTabletOp(); + tableTabletOp.setSingleOperations(singleOps); + tableTabletOp.setTabletId(tabletId); + + tableLsOp.addTabletOperation(tableTabletOp); + + if (isFirstEntry) { + tableId = tableParam.getTableId(); + originPartId = tableParam.getPartId(); + operationTimeout = tableParam.getObTable().getObTableOperationTimeout(); + subObTable = tableParam.getObTable(); + isFirstEntry = false; + } } - ObTableTabletOp tabletOp = new ObTableTabletOp(); - tabletOp.setSingleOperations(subOperations); - tabletOp.setTabletId(partId); - ObTableLSOperation lsOperation = new ObTableLSOperation(); - lsOperation.addTabletOperation(tabletOp); // Since we only have one tablet operation // We do the LS operation prepare here - lsOperation.prepare(); + tableLsOp.prepare(); - subRequest.setLsOperation(lsOperation); - subRequest.setTableId(tableId); - subRequest.setEntityType(entityType); - subRequest.setTimeout(subObTable.getObTableOperationTimeout()); + tableLsOpRequest.setLsOperation(tableLsOp); + tableLsOpRequest.setTableId(tableId); + tableLsOpRequest.setEntityType(entityType); + tableLsOpRequest.setTimeout(operationTimeout); ObTableLSOpResult subLSOpResult; boolean needRefreshTableEntry = false; @@ -302,10 +433,10 @@ public void partitionExecute(ObTableSingleOpResult[] results, long currentExecute = System.currentTimeMillis(); long costMillis = currentExecute - startExecute; if (costMillis > obTableClient.getRuntimeMaxWait()) { - logger.error("table name: {} partition id:{} it has tried " + tryTimes + logger.error("table name: {} ls id:{} it has tried " + tryTimes + " times and it has waited " + costMillis + " ms" + " which exceeds runtime max wait timeout " - + obTableClient.getRuntimeMaxWait() + " ms", tableName, partId); + + obTableClient.getRuntimeMaxWait() + " ms", tableName, lsId); throw new ObTableTimeoutExcetion("it has tried " + tryTimes + " times and it has waited " + costMillis + "ms which exceeds runtime max wait timeout " @@ -328,7 +459,7 @@ public void partitionExecute(ObTableSingleOpResult[] results, getRight().getObTable(); } } - subLSOpResult = (ObTableLSOpResult) subObTable.execute(subRequest); + subLSOpResult = (ObTableLSOpResult) subObTable.execute(tableLsOpRequest); obTableClient.resetExecuteContinuousFailureCount(tableName); break; } catch (Exception ex) { @@ -341,8 +472,8 @@ public void partitionExecute(ObTableSingleOpResult[] results, } } else if (ex instanceof ObTableReplicaNotReadableException) { if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { - logger.warn("tablename:{} partition id:{} retry when replica not readable: {}", - tableName, partId, ex.getMessage()); + logger.warn("tablename:{} ls id:{} retry when replica not readable: {}", + tableName, lsId, ex.getMessage()); if (failedServerList == null) { failedServerList = new HashSet(); } @@ -354,12 +485,12 @@ public void partitionExecute(ObTableSingleOpResult[] results, } else if (ex instanceof ObTableException && ((ObTableException) ex).isNeedRefreshTableEntry()) { needRefreshTableEntry = true; - logger.warn("tablename:{} partition id:{} batch ops refresh table while meet ObTableMasterChangeException, errorCode: {}", - tableName, partId, ((ObTableException) ex).getErrorCode(), ex); + logger.warn("tablename:{} ls id:{} batch ops refresh table while meet ObTableMasterChangeException, errorCode: {}", + tableName, lsId, ((ObTableException) ex).getErrorCode(), ex); if (obTableClient.isRetryOnChangeMasterTimes() && (tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { - logger.warn("tablename:{} partition id:{} batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", - tableName, partId, ((ObTableException) ex).getErrorCode(), + logger.warn("tablename:{} ls id:{} batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", + tableName, lsId, ((ObTableException) ex).getErrorCode(), tryTimes, ex); } else { obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); @@ -376,79 +507,101 @@ public void partitionExecute(ObTableSingleOpResult[] results, long endExecute = System.currentTimeMillis(); if (subLSOpResult == null) { - RUNTIME.error("tablename:{} partition id:{} check batch operation result error: client get unexpected NULL result", - tableName, partId); + RUNTIME.error("tablename:{} ls id:{} check batch operation result error: client get unexpected NULL result", + tableName, lsId); throw new ObTableUnexpectedException("check batch operation result error: client get unexpected NULL result"); } List tabletOpResults = subLSOpResult.getResults(); - if (tabletOpResults.size() != 1) { - throw new ObTableUnexpectedException("check batch result error: partition " - + partId + " expect tablet op result size 1" - + " actual result size is " - + tabletOpResults.size()); - } + int affectedRows = 0; - List subObTableSingleOpResults = tabletOpResults.get(0).getResults(); - - if (subObTableSingleOpResults.size() < subOperations.size()) { - // only one result when it across failed - // only one result when hkv puts - if (subObTableSingleOpResults.size() == 1 && entityType == ObTableEntityType.HKV) { - ObTableSingleOpResult subObTableSingleOpResult = subObTableSingleOpResults.get(0); - subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); - subObTableSingleOpResult.setExecutePort(subObTable.getPort()); - for (ObPair SubOperationWithIndexList : subOperationWithIndexList) { - results[SubOperationWithIndexList.getLeft()] = subObTableSingleOpResult; - } - } else { - throw new IllegalArgumentException( - "check batch operation result size error: operation size [" - + subOperations.size() + "] result size [" - + subObTableSingleOpResults.size() + "]"); + if (returnOneResult) { + if (results[0] == null) { + results[0] = new ObTableSingleOpResult(); } - } else { - if (subOperationWithIndexList.size() != subObTableSingleOpResults.size()) { - throw new ObTableUnexpectedException("check batch result error: partition " - + partId + " expect result size " - + subOperationWithIndexList.size() - + " actual result size " - + subObTableSingleOpResults.size()); + + ObTableSingleOpResult singleOpResult = tabletOpResults.get(0).getResults().get(0); + if (singleOpResult.getHeader().getErrno() != ResultCodes.OB_SUCCESS.errorCode) { + results[0].getHeader().setErrno(singleOpResult.getHeader().getErrno()); + results[0].getHeader().setMsg(singleOpResult.getHeader().getMsg()); } - for (int i = 0; i < subOperationWithIndexList.size(); i++) { - ObTableSingleOpResult subObTableSingleOpResult = subObTableSingleOpResults.get(i); - subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); - subObTableSingleOpResult.setExecutePort(subObTable.getPort()); - results[subOperationWithIndexList.get(i).getLeft()] = subObTableSingleOpResult; + results[0].setAffectedRows(results[0].getAffectedRows() + + tabletOpResults.get(0).getResults().get(0).getAffectedRows()); + } else { + for (int i = 0; i < tabletOpResults.size(); i++) { + List singleOpResults = tabletOpResults.get(i).getResults(); + for (int j = 0; j < singleOpResults.size(); j++) { + affectedRows += singleOpResults.size(); + } + List> singleOperationsWithIndexList = lsOperationWithIndexList.get(i); + if (singleOpResults.size() < singleOperationsWithIndexList.size()) { + // only one result when it across failed + // only one result when hkv puts + if (singleOpResults.size() == 1 && entityType == ObTableEntityType.HKV) { + ObTableSingleOpResult subObTableSingleOpResult = singleOpResults.get(0); + subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); + subObTableSingleOpResult.setExecutePort(subObTable.getPort()); + for (ObPair SubOperationWithIndexList : singleOperationsWithIndexList) { + results[SubOperationWithIndexList.getLeft()] = subObTableSingleOpResult; + } + } else { + throw new IllegalArgumentException( + "check batch operation result size error: operation size [" + + singleOperationsWithIndexList.size() + "] result size [" + + singleOpResults.size() + "]"); + } + } else { + if (singleOpResults.size() != singleOperationsWithIndexList.size()) { + throw new ObTableUnexpectedException("check batch result error: ls " + + lsId + " expect result size " + + singleOperationsWithIndexList.size() + + " actual result size " + + singleOpResults.size() + + " for " + i + "th tablet operation"); + } + for (int j = 0; j < singleOperationsWithIndexList.size(); j++) { + ObTableSingleOpResult subObTableSingleOpResult = singleOpResults.get(j); + subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); + subObTableSingleOpResult.setExecutePort(subObTable.getPort()); + results[singleOperationsWithIndexList.get(j).getLeft()] = subObTableSingleOpResult; + } + } } } + + String endpoint = subObTable.getIp() + ":" + subObTable.getPort(); - MonitorUtil.info(subRequest, subObTable.getDatabase(), tableName, - "BATCH-partitionExecute-", endpoint, tabletOp, - subObTableSingleOpResults.size(), endExecute - startExecute, + MonitorUtil.info(tableLsOpRequest, subObTable.getDatabase(), tableName, + "LS_BATCH-Execute-", endpoint, tableLsOp, + affectedRows, endExecute - startExecute, obTableClient.getslowQueryMonitorThreshold()); } /* * Execute internal. */ - public ObTableTabletOpResult executeInternal() throws Exception { + public ObTableSingleOpResult[] executeInternal() throws Exception { if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } long start = System.currentTimeMillis(); - final ObTableSingleOpResult[] obTableOperationResults = new ObTableSingleOpResult[batchOperation - .size()]; - Map>>> partitions = partitionPrepare(); + ObTableSingleOpResult[] obTableOperationResults = null; + if (returnOneResult) { + obTableOperationResults = new ObTableSingleOpResult[1]; + } else { + obTableOperationResults = new ObTableSingleOpResult[batchOperation.size()]; + } + Map>>>> lsOperations = partitionPrepare(); long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); - if (executorService != null && !executorService.isShutdown() && partitions.size() > 1) { + if (executorService != null && !executorService.isShutdown() && lsOperations.size() > 1) { // execute sub-batch operation in parallel final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, - partitions.size()); - for (final Map.Entry>>> entry : partitions + lsOperations.size()); + for (final Map.Entry>>>> entry : lsOperations .entrySet()) { + ObTableSingleOpResult[] finalObTableOperationResults = obTableOperationResults; executor.execute(new ConcurrentTask() { /* * Do task. @@ -457,7 +610,7 @@ public ObTableTabletOpResult executeInternal() throws Exception { public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - partitionExecute(obTableOperationResults, entry); + partitionExecute(finalObTableOperationResults, entry); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); @@ -504,23 +657,24 @@ public void doTask() { } else { // Execute sub-batch operation one by one - for (final Map.Entry>>> entry : partitions + for (final Map.Entry>>>> entry : lsOperations .entrySet()) { partitionExecute(obTableOperationResults, entry); } } - ObTableTabletOpResult batchOperationResult = new ObTableTabletOpResult(); - for (ObTableSingleOpResult obTableOperationResult : obTableOperationResults) { - batchOperationResult.addResult(obTableOperationResult); + if (obTableOperationResults.length <= 0) { + throw new ObTableUnexpectedException( + "Ls batch execute returns zero single operation results"); } - MonitorUtil.info(batchOperationResult, obTableClient.getDatabase(), tableName, "BATCH", "", - obTableOperationResults.length, getTableTime - start, System.currentTimeMillis() - - getTableTime, - obTableClient.getslowQueryMonitorThreshold()); + MonitorUtil + .info(obTableOperationResults[0], obTableClient.getDatabase(), tableName, "LS_BATCH", + "", obTableOperationResults.length, getTableTime - start, + System.currentTimeMillis() - getTableTime, + obTableClient.getslowQueryMonitorThreshold()); - return batchOperationResult; + return obTableOperationResults; } /* diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java index 1377e260..216382c3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java @@ -21,12 +21,14 @@ import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_TABLET_ID; import static com.alipay.oceanbase.rpc.protocol.payload.Constants.OB_INVALID_ID; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_LS_ID; public class ObTableParam { private ObTable obTable; private long tableId = OB_INVALID_ID; private long partitionId = INVALID_TABLET_ID; // partition id in 3.x aka tablet id in 4.x private long partId = INVALID_TABLET_ID; // logicId, partition id in 3.x, can be used when retry + private long lsId = INVALID_LS_ID; /* * constructor @@ -113,4 +115,19 @@ public long getPartId() { public void setPartId(long partId) { this.partId = partId; } + + /* + * Set lsId + */ + public long getLsId() { + return lsId; + } + + /* + * Get lsId + */ + public void setLsId(long lsId) { + this.lsId = lsId; + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java index efd0bf39..ff0d0214 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java @@ -293,22 +293,22 @@ public static void info(final ObPayload payload, String database, String tableNa /** * for tablet op */ - private static void logTabletOpMessage(final ObPayload payload, String database, - String tableName, String methodName, String endpoint, - ObTableTabletOp tabletOp, int resultSize, - long executeTime, long slowQueryMonitorThreshold) { + private static void logLsOpMessage(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, + ObTableLSOperation lsOperation, int resultSize, + long executeTime, long slowQueryMonitorThreshold) { if (executeTime < slowQueryMonitorThreshold) { return; } String traceId = formatTraceMessage(payload); MONITOR.info(logMessage(traceId, database, tableName, - methodName + "-" + tabletOp.getTabletId(), endpoint, null, resultSize, executeTime)); + methodName + "-" + lsOperation.getLsId(), endpoint, null, resultSize, executeTime)); } public static void info(final ObPayload payload, String database, String tableName, - String methodName, String endpoint, ObTableTabletOp tabletOp, + String methodName, String endpoint, ObTableLSOperation lsOperation, int resultSize, long executeTime, long slowQueryMonitorThreshold) { - logTabletOpMessage(payload, database, tableName, methodName, endpoint, tabletOp, - resultSize, executeTime, slowQueryMonitorThreshold); + logLsOpMessage(payload, database, tableName, methodName, endpoint, lsOperation, resultSize, + executeTime, slowQueryMonitorThreshold); } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index d38e9629..8c709a7c 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -112,9 +112,10 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false { batchOps.clear(); - batchOps.insert("abc-1", new String[]{"c1", "c2"}, new String[]{"bar-1", "bar-2"}); - batchOps.get("abc-2", new String[]{"c2"}); - batchOps.insert("abc-3", new String[]{"c2"}, new String[]{"bar-3"}); + batchOps.insert("abc-1", new String[] { "c1", "c2" }, new String[] { "bar-1", + "bar-2" }); + batchOps.get("abc-2", new String[] { "c2" }); + batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); @@ -123,18 +124,18 @@ public void testBatchOperation() { // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: false { batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); - batchOps.get("abc-4", new String[]{"c1", "c2"}); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); + batchOps.get("abc-4", new String[] { "c1", "c2" }); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); - batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); + batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); @@ -143,9 +144,9 @@ public void testBatchOperation() { // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: true { batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); @@ -154,11 +155,13 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: true { batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1a", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c2", "c3", "C1A"}); - batchOps.get("abc-4", new String[]{"c1A", "c2", "c3"}); - batchOps.insert("abc-4", new String[]{"c3", "C2", "c1a"}, new String[]{"bar-3", "bar-3", "bar-3"}); - batchOps.insert("abc-4", new String[]{"c1A", "c2", "C3"}, new String[]{"bar-2", "bar-2", "bar-2"}); + batchOps.get("abc-2", new String[] { "c1a", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c2", "c3", "C1A" }); + batchOps.get("abc-4", new String[] { "c1A", "c2", "c3" }); + batchOps.insert("abc-4", new String[] { "c3", "C2", "c1a" }, new String[] { + "bar-3", "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c1A", "c2", "C3" }, new String[] { + "bar-2", "bar-2", "bar-2" }); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); @@ -167,21 +170,25 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false { batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-4", new String[]{"c1", "c2"}); - batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", "bar-3", "bar-3"}); - batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-2", "bar-2", "bar-2"}); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-4", new String[] { "c1", "c2" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", + "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-2", + "bar-2", "bar-2" }); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); batchOps.clear(); - batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); - batchOps.get("abc-3", new String[]{"c1", "c4", "c3"}); - batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", "bar-3", "bar-3"}); - batchOps.insert("abc-4", new String[]{"c2", "c3", "c1"}, new String[]{"bar-2", "bar-2", "bar-2"}); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c4", "c3" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", + "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c2", "c3", "c1" }, new String[] { "bar-2", + "bar-2", "bar-2" }); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java index 581ecc18..58f36fab 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java @@ -1940,6 +1940,9 @@ public void testPut() throws Exception { @Test public void testBatchMutation() throws Exception { + if (ObGlobal.isLsOpSupport()) { + return; + } System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); TableQuery tableQuery = client.query("test_mutation"); @@ -2230,7 +2233,6 @@ public void testAtomicBatchMutation() throws Exception { .addMutateColVal(colVal("c4", 100L)); BatchOperationResult result = batchOperation.addOperation(insert_0, insert_1) .setIsAtomic(true).execute(); - assertTrue(false); } catch (Exception e) { e.printStackTrace(); if (client instanceof ObTableClient && ((ObTableClient) client).isOdpMode()) { diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java new file mode 100644 index 00000000..6b1b80dc --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java @@ -0,0 +1,722 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import com.alipay.oceanbase.rpc.util.TimeUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.*; + +public class ObTableLsBatchTest { + public ObTableClient client; + private static final String TABLE_NAME = "test_mutation"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + } + + /* + CREATE TABLE `test_mutation` ( + `c1` bigint NOT NULL, + `c2` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + @Test + public void testInsertUp() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insertOrUpdate.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insertOrUpdate); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testGet() throws Exception { + // prepare data + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insertOrUpdate.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + MutationResult res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + } + + try { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey( + row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))).select("c1", "c2", "c3", + "c4"); + batchOperation.addOperation(query); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Row row = batchOperationResult.get(j).getOperationRow(); + Assert.assertEquals(values[j][0], row.get("c1")); + Assert.assertEquals(values[j][1], row.get("c2")); + Assert.assertEquals(values[j][2], new String((byte[]) row.get("c3"), "UTF-8")); + Assert.assertEquals(values[j][3], row.get("c4")); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + /* + CREATE TABLE IF NOT EXISTS `test_table_object` ( + `c1` tinyint primary key, + `c2` smallint not null, + `c3` int not null, + `c4` bigint not null, + `c5` varchar(128) not null, + `c6` varbinary(128) not null, + `c7` float not null, + `c8` double not null, + `c9` timestamp(6) not null, + `c10` datetime(6) not null, + `c11` int default null + ); + */ + @Test + public void testGetAllObjType() throws Exception { + String ALL_OBJ_TYPE_TABLE = "test_table_object"; + + // pre-clean data + client.delete(ALL_OBJ_TYPE_TABLE).addScanRange(ObObj.getMin(), ObObj.getMax()).execute(); + + long timeInMillis = System.currentTimeMillis(); + Timestamp c9Val = new Timestamp(timeInMillis); + Date c10Val = TimeUtils.strToDate("2024-01-30"); + + // prepare data + Object values[][] = { + {(byte)1, (short)1, (int)1, 1L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, + {(byte)2, (short)2, (int)2, 2L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, + {(byte)3, (short)3, (int)3, 3L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, + {(byte)4, (short)4, (int)4, 4L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, + {(byte)5, (short)5, (int)5, 5L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, + {(byte)6, (short)6, (int)6, 6L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null} + }; + + int rowCnt = values.length; + + try { + // pre insert data + { + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]))); + + for (int j = 2; j <= curRow.length; j++) { + insertOrUpdate.addMutateRow(row(colVal("c" + j, curRow[j-1]))); + } + batchOperation.addOperation(insertOrUpdate); + } + BatchOperationResult res = batchOperation.execute(); + for (int k = 0; k < rowCnt; k++) { + Assert.assertEquals(1, res.get(k).getAffectedRows()); + } + } + + // get data with all columns + { + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey(row(colVal("c1", curRow[0]))) + .select("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11"); + batchOperation.addOperation(query); + } + BatchOperationResult res = batchOperation.execute(); + + for (int j = 0; j < rowCnt; j++) { + Object[] curRow = values[j]; + Row row = res.get(j).getOperationRow(); + for (int k = 0; k < curRow.length; k++) { + Object getValue = row.get("c" + (k+1)); + if (getValue instanceof byte[]) { + Assert.assertTrue(Arrays.equals((byte[]) curRow[k], (byte[]) getValue)); + } else { + Assert.assertEquals(curRow[k], getValue); + } + } + } + } + + // get data with different columns + { + // get columns idx, 1 means get c1 + int columns[][] = { + {7, 3, 11, 5, 6, 2, 9, 1, 8, 4, 10}, + {5, 2, 7, 9, 1, 8, 6, 3, 10, 4}, + {5, 3, 1, 8, 6, 4, 7, 2, 9}, + {3, 2, 5, 8, 7, 1, 6, 4}, + {4, 6, 7, 1, 3, 5, 2}, + {2, 6, 3, 1, 4, 5}, + }; + + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey(row(colVal("c1", curRow[0]))); + List selectColumns = new ArrayList<>(); + for (int j = 0; j < columns[i].length; j++) { + selectColumns.add("c" + columns[i][j]); + } + query.select(selectColumns.toArray(new String[0])); + batchOperation.addOperation(query); + } + BatchOperationResult res = batchOperation.execute(); + + for (int j = 0; j < rowCnt; j++) { + int curColumns[] = columns[j]; + Object[] curRow = values[j]; + Row row = res.get(j).getOperationRow(); + for (int k = 0; k < curColumns.length; k++) { + Object curValue = curRow[curColumns[k]-1]; + String curSelectColumn = "c" + curColumns[k]; + Object getValue = row.get(curSelectColumn); + if (getValue instanceof byte[]) { + Assert.assertTrue(Arrays.equals((byte[]) curValue, (byte[]) getValue)); + } else { + Assert.assertEquals(curValue, getValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally {} + } + + @Test + public void testBatchInsert() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchAppend() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // append + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Append append = new Append(); + append.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + append.addMutateRow(row(colVal("c3", curRow[2]))); + batchOperation.addOperation(append); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchDel() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // del + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Delete delete = new Delete(); + delete.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + batchOperation.addOperation(delete); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + // todo: the multi-delete's affected_rows is inaccuracy cuz of server's bug + // Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + } + + @Test + public void testBatchIncrement() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // increment + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Increment inc = new Increment(); + inc.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + inc.addMutateRow(row(colVal("c4", curRow[3]))); + batchOperation.addOperation(inc); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchReplace() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // replace + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Replace replace = new Replace(); + replace.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + replace.addMutateRow(row(colVal("c3", curRow[1]), colVal("c4", curRow[3]))); + batchOperation.addOperation(replace); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt * 2, batchOperationResult.get(0).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchUpdate() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // update + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Update update = new Update(); + update.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + update.addMutateRow(row(colVal("c3", curRow[1]), colVal("c4", curRow[3]))); + batchOperation.addOperation(update); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testHybridBatch() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + + try { + // 1. insert {1L, "c2_val", "c3_val", 100L} + { + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + insert.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 100L))); + batchOperation.addOperation(insert); + } + + // 2. append {1L, "c2_val", "_val"} + { + Append append = new Append(); + append.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + append.addMutateRow(row(colVal("c3", "_val"))); + batchOperation.addOperation(append); + } + + // 3. get {1L, "c2_val"} + { + TableQuery query1 = query() + .setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))).select("c1", "c2", + "c3", "c4"); + batchOperation.addOperation(query1); + } + + // 4. delete {1L, "c2_val"} + { + Delete delete = new Delete(); + delete.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + batchOperation.addOperation(delete); + + } + + // 5. increment {1001L, "c2_val", 100L} + { + Increment inc = new Increment(); + inc.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + inc.addMutateRow(row(colVal("c4", 100L))); + batchOperation.addOperation(inc); + } + + // 6. insertUp {1001L, "c2_val", "c3_val", 100L} + { + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 200L))); + batchOperation.addOperation(insertOrUpdate); + } + + // 7. replace {1001L, "c2_val", "c3_val", 300L} + { + Replace replace = new Replace(); + replace.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + replace.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 300L))); + batchOperation.addOperation(replace); + } + + // 8. update {1001L, "c2_val", "c3_val_val", 400L} + { + Update update = new Update(); + update.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + update.addMutateRow(row(colVal("c3", "c3_val_val"), colVal("c4", 400L))); + batchOperation.addOperation(update); + } + + // 9. get {1001L, "c2_val"} + { + TableQuery query2 = query().setRowKey( + row(colVal("c1", 1001L), colVal("c2", "c2_val"))) + .select("c1", "c2", "c3", "c4"); + batchOperation.addOperation(query2); + } + + BatchOperationResult result = batchOperation.execute(); + + int affectRows[] = { 1, 1, 0, 1, 1, 1, 2, 1, 0 }; + for (int i = 0; i < affectRows.length; i++) { + Assert.assertEquals(affectRows[i], result.get(i).getAffectedRows()); + } + + Object getExpRows[][] = { { 2 /*res idx*/, 1L, "c2_val", "c3_val_val", 100L }, + { 8 /*res idx*/, 1001L, "c2_val", "c3_val_val", 400L } }; + + for (int j = 0; j < getExpRows.length; j++) { + Object curRow[] = getExpRows[j]; + Row row = result.get((int) curRow[0]).getOperationRow(); + Assert.assertEquals(curRow[1], row.get("c1")); + Assert.assertEquals(curRow[2], row.get("c2")); + Assert.assertEquals(curRow[3], new String((byte[]) row.get("c3"), "UTF-8")); + Assert.assertEquals(curRow[4], row.get("c4")); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + Object rowkeys[][] = { { 1L, "c2_val" }, { 1001L, "c2_val" }, }; + for (int i = 0; i < rowkeys.length; i++) { + Object curRow[] = rowkeys[i]; + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + delete.execute(); + } + } + } + + @Test + public void testPut() throws Exception { + // put operation should set binlog_row_image minimal + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image= 'minimal'"); + + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 200L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 2000L, "c2_val", "c3_val", 100L }, + { 100001L, "c2_val", "c3_val", 100L }, { 10000002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Put put = new Put(); + put.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + put.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(put); + } + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + statement.execute("SET GLOBAL binlog_row_image= 'FULL'"); + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java index 308c426d..653046ef 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java @@ -43,7 +43,7 @@ **/ public class ObTablePutTest { ObTableClient client; - public static String tableName = "test_put"; + public static String tableName = "test_put"; @Before public void setup() throws Exception { @@ -72,21 +72,15 @@ public void testPut1() throws Exception { Timestamp curTs = new Timestamp(System.currentTimeMillis()); curTs.setNanos(0); client.put(tableName).setRowKey(colVal("id", "id0")) - .addMutateColVal(colVal("expired_ts", curTs)) - .execute(); + .addMutateColVal(colVal("expired_ts", curTs)).execute(); + client.put(tableName).setRowKey(colVal("id", "id0")).addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("expired_ts", curTs)).execute(); + client.put(tableName).setRowKey(colVal("id", "id0")).addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)).addMutateColVal(colVal("expired_ts", curTs)) + .execute(); client.put(tableName).setRowKey(colVal("id", "id0")) - .addMutateColVal(colVal("c1", 1L)) - .addMutateColVal(colVal("expired_ts", curTs)) - .execute(); - client.put(tableName).setRowKey(colVal("id", "id0")) - .addMutateColVal(colVal("c1", 1L)) - .addMutateColVal(colVal("c2", 2L)) - .addMutateColVal(colVal("expired_ts", curTs)) - .execute(); - client.put(tableName).setRowKey(colVal("id", "id0")) - .addMutateColVal(colVal("c3", "c3")) - .addMutateColVal(colVal("expired_ts", curTs)) - .execute(); + .addMutateColVal(colVal("c3", "c3")).addMutateColVal(colVal("expired_ts", curTs)) + .execute(); } catch (Exception e) { e.printStackTrace(); Assert.assertTrue(false); From 33e132e925b4d7fd59ae9c5f91a7238ab57d2bac Mon Sep 17 00:00:00 2001 From: eemjwu <34029771+eemjwu@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:37:33 +0800 Subject: [PATCH 10/11] fix return_one_result flag not return one result (#110) * fix return_one_result flag not return one result * fix review --- .../oceanbase/rpc/ObClusterTableBatchOps.java | 22 +- .../rpc/table/ObTableClientBatchOpsImpl.java | 80 +++++--- .../rpc/ObAtomicBatchOperationTest.java | 194 ++++++++++++------ 3 files changed, 200 insertions(+), 96 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java index 43db7d05..20f0c724 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java @@ -17,9 +17,7 @@ package com.alipay.oceanbase.rpc; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperation; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperationResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; import com.alipay.oceanbase.rpc.table.AbstractTableBatchOps; import com.alipay.oceanbase.rpc.table.ObTableClientBatchOpsImpl; @@ -117,6 +115,7 @@ public void put(Object[] rowkeys, String[] columns, Object[] values) { */ @Override public List execute() throws Exception { + preCheck(); return tableBatchOps.execute(); } @@ -125,6 +124,7 @@ public List execute() throws Exception { */ @Override public List executeWithResult() throws Exception { + preCheck(); return tableBatchOps.executeWithResult(); } @@ -132,6 +132,7 @@ public List executeWithResult() throws Exception { * Execute internal. */ public ObTableBatchOperationResult executeInternal() throws Exception { + preCheck(); return tableBatchOps.executeInternal(); } @@ -181,4 +182,19 @@ public void setReturnOneResult(boolean returnOneResult) { super.setReturnOneResult(returnOneResult); tableBatchOps.setReturnOneResult(returnOneResult); } + + void preCheck() { + List operations = this.tableBatchOps.getObTableBatchOperation().getTableOperations(); + if (operations.isEmpty()) { + throw new IllegalArgumentException("operations is empty"); + } + ObTableOperationType lastType = operations.get(0).getOperationType(); + if (returnOneResult + && !(this.tableBatchOps.getObTableBatchOperation().isSameType() && (lastType == ObTableOperationType.INSERT + || lastType == ObTableOperationType.PUT + || lastType == ObTableOperationType.REPLACE || lastType == ObTableOperationType.DEL))) { + throw new IllegalArgumentException( + "returnOneResult only support multi-insert/put/replace/del"); + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index a0648fb0..70d3497b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -406,38 +406,50 @@ public void partitionExecute(ObTableOperationResult[] results, List subObTableOperationResults = subObTableBatchOperationResult .getResults(); - if (subObTableOperationResults.size() < subOperations.getTableOperations().size()) { - // only one result when it across failed - // only one result when hkv puts - if (subObTableOperationResults.size() == 1) { - ObTableOperationResult subObTableOperationResult = subObTableOperationResults - .get(0); + if (returnOneResult) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults.get(0); + if (results[0] == null) { + results[0] = new ObTableOperationResult(); subObTableOperationResult.setExecuteHost(subObTable.getIp()); subObTableOperationResult.setExecutePort(subObTable.getPort()); - for (ObPair aSubOperationWithIndexList : subOperationWithIndexList) { - results[aSubOperationWithIndexList.getLeft()] = subObTableOperationResult; - } + results[0] = subObTableOperationResult; } else { - // unexpected result found - throw new IllegalArgumentException( - "check batch operation result size error: operation size [" - + subOperations.getTableOperations().size() + "] result size [" - + subObTableOperationResults.size() + "]"); + results[0].setAffectedRows(results[0].getAffectedRows() + subObTableOperationResult.getAffectedRows()); } } else { - if (subOperationWithIndexList.size() != subObTableOperationResults.size()) { - throw new ObTableUnexpectedException("check batch result error: partition " - + partId + " expect result size " - + subOperationWithIndexList.size() - + " actual result size " - + subObTableOperationResults.size()); - } - for (int i = 0; i < subOperationWithIndexList.size(); i++) { - ObTableOperationResult subObTableOperationResult = subObTableOperationResults - .get(i); - subObTableOperationResult.setExecuteHost(subObTable.getIp()); - subObTableOperationResult.setExecutePort(subObTable.getPort()); - results[subOperationWithIndexList.get(i).getLeft()] = subObTableOperationResult; + if (subObTableOperationResults.size() < subOperations.getTableOperations().size()) { + // only one result when it across failed + // only one result when hkv puts + if (subObTableOperationResults.size() == 1) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults + .get(0); + subObTableOperationResult.setExecuteHost(subObTable.getIp()); + subObTableOperationResult.setExecutePort(subObTable.getPort()); + for (ObPair aSubOperationWithIndexList : subOperationWithIndexList) { + results[aSubOperationWithIndexList.getLeft()] = subObTableOperationResult; + } + } else { + // unexpected result found + throw new IllegalArgumentException( + "check batch operation result size error: operation size [" + + subOperations.getTableOperations().size() + "] result size [" + + subObTableOperationResults.size() + "]"); + } + } else { + if (subOperationWithIndexList.size() != subObTableOperationResults.size()) { + throw new ObTableUnexpectedException("check batch result error: partition " + + partId + " expect result size " + + subOperationWithIndexList.size() + + " actual result size " + + subObTableOperationResults.size()); + } + for (int i = 0; i < subOperationWithIndexList.size(); i++) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults + .get(i); + subObTableOperationResult.setExecuteHost(subObTable.getIp()); + subObTableOperationResult.setExecutePort(subObTable.getPort()); + results[subOperationWithIndexList.get(i).getLeft()] = subObTableOperationResult; + } } } String endpoint = subObTable.getIp() + ":" + subObTable.getPort(); @@ -457,8 +469,13 @@ public ObTableBatchOperationResult executeInternal() throws Exception { } long start = System.currentTimeMillis(); List operations = batchOperation.getTableOperations(); - final ObTableOperationResult[] obTableOperationResults = new ObTableOperationResult[operations - .size()]; + ObTableOperationResult[] obTableOperationResults = null; + if (returnOneResult) { + obTableOperationResults = new ObTableOperationResult[1]; + } else { + obTableOperationResults = new ObTableOperationResult[operations.size()]; + } + Map>>> partitions = partitionPrepare(); long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); @@ -466,7 +483,8 @@ public ObTableBatchOperationResult executeInternal() throws Exception { final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, partitions.size()); for (final Map.Entry>>> entry : partitions - .entrySet()) { + .entrySet()) { + ObTableOperationResult[] finalObTableOperationResults = obTableOperationResults; executor.execute(new ConcurrentTask() { /* * Do task. @@ -475,7 +493,7 @@ public ObTableBatchOperationResult executeInternal() throws Exception { public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - partitionExecute(obTableOperationResults, entry); + partitionExecute(finalObTableOperationResults, entry); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index 8c709a7c..29036c41 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -19,6 +19,12 @@ import com.alipay.oceanbase.rpc.exception.ObTableDuplicateKeyException; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.BatchOperation; +import com.alipay.oceanbase.rpc.mutation.Delete; +import com.alipay.oceanbase.rpc.mutation.Insert; +import com.alipay.oceanbase.rpc.mutation.Put; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import org.junit.After; @@ -26,15 +32,19 @@ import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; + public class ObAtomicBatchOperationTest { - private static final int dataSetSize = 4; - private static final String successKey = "abc-5"; - private static final String failedKey = "abc-7"; + private static final int dataSetSize = 4; + private static final String successKey = "abc-5"; + private static final String failedKey = "abc-7"; - protected ObTableClient obTableClient; + protected ObTableClient obTableClient; @Before public void setup() throws Exception { @@ -51,8 +61,8 @@ public void setup() throws Exception { for (int i = 0; i < dataSetSize; i++) { String key = "abc-" + i; String val = "xyz-" + i; - this.obTableClient.insert("test_varchar_table", key, new String[] { "c2" }, - new String[] { val }); + this.obTableClient.insert("test_varchar_table", key, new String[]{"c2"}, + new String[]{val}); } } @@ -76,10 +86,10 @@ public void testAtomic() { // default: no atomic batch operation try { batchOps.clear(); - batchOps.insert("abc-1", new String[] { "c2" }, new String[] { "bar-1" }); - batchOps.get("abc-2", new String[] { "c2" }); - batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); - batchOps.insert(successKey, new String[] { "c2" }, new String[] { "bar-5" }); + batchOps.insert("abc-1", new String[]{"c2"}, new String[]{"bar-1"}); + batchOps.get("abc-2", new String[]{"c2"}); + batchOps.insert("abc-3", new String[]{"c2"}, new String[]{"bar-3"}); + batchOps.insert(successKey, new String[]{"c2"}, new String[]{"bar-5"}); List results = batchOps.execute(); Assert.assertTrue(results.get(0) instanceof ObTableException); Assert.assertEquals(((Map) results.get(1)).get("c2"), "xyz-2"); @@ -93,10 +103,10 @@ public void testAtomic() { try { batchOps.clear(); batchOps.setAtomicOperation(true); - batchOps.insert("abc-1", new String[] { "c2" }, new String[] { "bar-1" }); - batchOps.get("abc-2", new String[] { "c2" }); - batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); - batchOps.insert(failedKey, new String[] { "c2" }, new String[] { "bar-5" }); + batchOps.insert("abc-1", new String[]{"c2"}, new String[]{"bar-1"}); + batchOps.get("abc-2", new String[]{"c2"}); + batchOps.insert("abc-3", new String[]{"c2"}, new String[]{"bar-3"}); + batchOps.insert(failedKey, new String[]{"c2"}, new String[]{"bar-5"}); batchOps.execute(); // no support atomic batch // Assert.fail("expect duplicate key exception."); @@ -112,10 +122,10 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false { batchOps.clear(); - batchOps.insert("abc-1", new String[] { "c1", "c2" }, new String[] { "bar-1", - "bar-2" }); - batchOps.get("abc-2", new String[] { "c2" }); - batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); + batchOps.insert("abc-1", new String[]{"c1", "c2"}, new String[]{"bar-1", + "bar-2"}); + batchOps.get("abc-2", new String[]{"c2"}); + batchOps.insert("abc-3", new String[]{"c2"}, new String[]{"bar-3"}); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); @@ -124,18 +134,18 @@ public void testBatchOperation() { // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: false { batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); - batchOps.get("abc-4", new String[] { "c1", "c2" }); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); + batchOps.get("abc-4", new String[]{"c1", "c2"}); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); - batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c4"}); + batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); @@ -144,9 +154,9 @@ public void testBatchOperation() { // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: true { batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-4", new String[]{"c1", "c2", "c3"}); Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); @@ -155,13 +165,13 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: true { batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1a", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c2", "c3", "C1A" }); - batchOps.get("abc-4", new String[] { "c1A", "c2", "c3" }); - batchOps.insert("abc-4", new String[] { "c3", "C2", "c1a" }, new String[] { - "bar-3", "bar-3", "bar-3" }); - batchOps.insert("abc-4", new String[] { "c1A", "c2", "C3" }, new String[] { - "bar-2", "bar-2", "bar-2" }); + batchOps.get("abc-2", new String[]{"c1a", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c2", "c3", "C1A"}); + batchOps.get("abc-4", new String[]{"c1A", "c2", "c3"}); + batchOps.insert("abc-4", new String[]{"c3", "C2", "c1a"}, new String[]{ + "bar-3", "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c1A", "c2", "C3"}, new String[]{ + "bar-2", "bar-2", "bar-2"}); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); @@ -170,25 +180,25 @@ public void testBatchOperation() { // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false { batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-4", new String[] { "c1", "c2" }); - batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", - "bar-3", "bar-3" }); - batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-2", - "bar-2", "bar-2" }); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-4", new String[]{"c1", "c2"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", + "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-2", + "bar-2", "bar-2"}); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); batchOps.clear(); - batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); - batchOps.get("abc-3", new String[] { "c1", "c4", "c3" }); - batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", - "bar-3", "bar-3" }); - batchOps.insert("abc-4", new String[] { "c2", "c3", "c1" }, new String[] { "bar-2", - "bar-2", "bar-2" }); + batchOps.get("abc-2", new String[]{"c1", "c2", "c3"}); + batchOps.get("abc-3", new String[]{"c1", "c4", "c3"}); + batchOps.insert("abc-4", new String[]{"c1", "c2", "c3"}, new String[]{"bar-3", + "bar-3", "bar-3"}); + batchOps.insert("abc-4", new String[]{"c2", "c3", "c1"}, new String[]{"bar-2", + "bar-2", "bar-2"}); Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); @@ -209,14 +219,12 @@ public void testReturnOneRes() { batchOps.clear(); batchOps.setAtomicOperation(false); batchOps.setReturnOneResult(true); - batchOps.insert("abcd-7", new String[] { "c2" }, new String[] { "returnOne-7" }); - batchOps.insert("abcd-8", new String[] { "c2" }, new String[] { "returnOne-8" }); - batchOps.insert("abcd-9", new String[] { "c2" }, new String[] { "returnOne-9" }); + batchOps.insert("abcd-7", new String[]{"c2"}, new String[]{"returnOne-7"}); + batchOps.insert("abcd-8", new String[]{"c2"}, new String[]{"returnOne-8"}); + batchOps.insert("abcd-9", new String[]{"c2"}, new String[]{"returnOne-9"}); List results = batchOps.execute(); - Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), 3L); - Assert.assertEquals(results.get(1), 3L); - Assert.assertEquals(results.get(2), 3L); } catch (Exception ex) { Assert.assertTrue(ex instanceof ObTableException); } @@ -226,14 +234,46 @@ public void testReturnOneRes() { batchOps.clear(); batchOps.setAtomicOperation(true); batchOps.setReturnOneResult(true); - batchOps.insert("abcd-4", new String[] { "c2" }, new String[] { "returnOne-4" }); - batchOps.insert("abcd-5", new String[] { "c2" }, new String[] { "returnOne-5" }); - batchOps.insert("abcd-6", new String[] { "c2" }, new String[] { "returnOne-6" }); + batchOps.insert("abcd-4", new String[]{"c2"}, new String[]{"returnOne-4"}); + batchOps.insert("abcd-5", new String[]{"c2"}, new String[]{"returnOne-5"}); + batchOps.insert("abcd-6", new String[]{"c2"}, new String[]{"returnOne-6"}); List results = batchOps.execute(); - Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), 3L); - Assert.assertEquals(results.get(1), 3L); - Assert.assertEquals(results.get(2), 3L); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + + // batch delete operation + try { + // del + batchOps.clear(); + batchOps.setReturnOneResult(true); + batchOps.setAtomicOperation(false); + for (int i = 0; i <= 9; i += 2) { + String key = "abc-" + i; + batchOps.delete(key); + } + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), 2L); + // get + batchOps.clear(); + batchOps.setAtomicOperation(true); + for (int i = 0; i <= 9; i++) { + String key = "abc-" + i; + batchOps.get(key, new String[]{"c2"}); + } + batchOps.setReturnOneResult(false); + List results_get = batchOps.execute(); + + Assert.assertTrue(((Map) results_get.get(0)).isEmpty()); + Assert.assertFalse(((Map) results_get.get(1)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(2)).isEmpty()); + Assert.assertFalse(((Map) results_get.get(3)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(4)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(5)).isEmpty()); + } catch (Exception ex) { Assert.assertTrue(ex instanceof ObTableException); } @@ -243,9 +283,9 @@ public void testReturnOneRes() { batchOps.clear(); batchOps.setAtomicOperation(false); batchOps.setReturnOneResult(false); - batchOps.insert("abcd-1", new String[] { "c2" }, new String[] { "returnOne-1" }); - batchOps.insert("abcd-2", new String[] { "c2" }, new String[] { "returnOne-2" }); - batchOps.insert("abcd-3", new String[] { "c2" }, new String[] { "returnOne-3" }); + batchOps.insert("abcd-1", new String[]{"c2"}, new String[]{"returnOne-1"}); + batchOps.insert("abcd-2", new String[]{"c2"}, new String[]{"returnOne-2"}); + batchOps.insert("abcd-3", new String[]{"c2"}, new String[]{"returnOne-3"}); List results = batchOps.execute(); Assert.assertEquals(results.size(), 3); Assert.assertEquals(results.get(0), 1L); @@ -255,4 +295,34 @@ public void testReturnOneRes() { Assert.fail("hit exception:" + ex); } } + + @Test + public void testReturnOneResPartition() throws Exception { + BatchOperation batchOperation = obTableClient.batchOperation("test_mutation"); + Object values[][] = {{1L, "c2_val", "c3_val", 100L}, {200L, "c2_val", "c3_val", 100L}, + {401L, "c2_val", "c3_val", 100L}, {2000L, "c2_val", "c3_val", 100L}, + {100001L, "c2_val", "c3_val", 100L}, {10000002L, "c2_val", "c3_val", 100L},}; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(batchOperationResult.size(), 1); + Assert.assertEquals(batchOperationResult.get(0).getAffectedRows(), 6); + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } finally { + for (int j = 0; j < rowCnt; j++) { + Object[] curRow = values[j]; + obTableClient.delete("test_mutation", new Object[]{curRow[0], curRow[1]}); + + } + } + } } From f0c7722947173046eacdf58bd1d7aa42d76976c3 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Mon, 18 Mar 2024 17:27:42 +0800 Subject: [PATCH 11/11] [Fix] adapt to lob-related table object type --- .../protocol/payload/impl/ObTableObjType.java | 113 +++++++++++++----- .../oceanbase/rpc/ObTableLsBatchTest.java | 47 ++++++-- src/test/resources/ci.sql | 11 +- 3 files changed, 129 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java index 62765a85..0a00e69b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java @@ -141,11 +141,50 @@ public int getEncodedSize(ObObj obj) { // 15 ObTableUInt32Type // 16 ObTableUInt64Type - ObTableInvalidType(17) { + ObTableTinyTextType(17) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableTextType(18) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableMediumTextType(19) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableLongTextType(20) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableTinyBlobType(21) { + }, + + ObTableBlobType(22) { + }, + + ObTableMediumBlobType(23) { + }, + + ObTableLongBlobType(24) { + }, + + ObTableInvalidType(25) { }; private int value; - private static Map map = new HashMap(); + // mapping from value to enum + private static Map valueMap = new HashMap(); + // mapping from ObTableObjType to ObObjType + private static Map tableObjTypeMap = new HashMap<>(); ObTableObjType(int value) { this.value = value; @@ -153,7 +192,7 @@ public int getEncodedSize(ObObj obj) { static { for (ObTableObjType type : ObTableObjType.values()) { - map.put(type.value, type); + valueMap.put(type.value, type); } } @@ -164,7 +203,7 @@ public static ObTableObjType getTableObjType(ObObj obj) { // only for GET operation default value return ObTableNullType; } else if (objType == ObObjType.ObTinyIntType) { - return ObTableObjType.ObTableTinyIntType; + return ObTableTinyIntType; } else if (objType == ObObjType.ObSmallIntType) { return ObTableObjType.ObTableSmallIntType; } else if (objType == ObObjType.ObInt32Type) { @@ -197,42 +236,44 @@ public static ObTableObjType getTableObjType(ObObj obj) { + objType.getClass().getName()); } + static { + tableObjTypeMap.put(ObTableNullType, ObObjType.ObNullType); + tableObjTypeMap.put(ObTableTinyIntType, ObObjType.ObTinyIntType); + tableObjTypeMap.put(ObTableSmallIntType, ObObjType.ObSmallIntType); + tableObjTypeMap.put(ObTableInt32Type, ObObjType.ObInt32Type); + tableObjTypeMap.put(ObTableInt64Type, ObObjType.ObInt64Type); + tableObjTypeMap.put(ObTableVarcharType, ObObjType.ObVarcharType); + tableObjTypeMap.put(ObTableVarbinaryType, ObObjType.ObVarcharType); + tableObjTypeMap.put(ObTableDoubleType, ObObjType.ObDoubleType); + tableObjTypeMap.put(ObTableFloatType, ObObjType.ObFloatType); + tableObjTypeMap.put(ObTableTimestampType, ObObjType.ObTimestampType); + tableObjTypeMap.put(ObTableDateTimeType, ObObjType.ObDateTimeType); + tableObjTypeMap.put(ObTableTinyTextType, ObObjType.ObTinyTextType); + tableObjTypeMap.put(ObTableTextType, ObObjType.ObTextType); + tableObjTypeMap.put(ObTableMediumTextType, ObObjType.ObMediumTextType); + tableObjTypeMap.put(ObTableLongTextType, ObObjType.ObLongTextType); + tableObjTypeMap.put(ObTableTinyBlobType, ObObjType.ObTinyTextType); + tableObjTypeMap.put(ObTableBlobType, ObObjType.ObTextType); + tableObjTypeMap.put(ObTableMediumBlobType, ObObjType.ObMediumTextType); + tableObjTypeMap.put(ObTableLongBlobType, ObObjType.ObLongTextType); + tableObjTypeMap.put(ObTableMinType, ObObjType.ObExtendType); + tableObjTypeMap.put(ObTableMaxType, ObObjType.ObExtendType); + } + public static ObObjType getObjType(ObTableObjType tableObjType) { - if (tableObjType == ObTableNullType) { - return ObObjType.ObNullType; - } else if (tableObjType == ObTableTinyIntType) { - return ObObjType.ObTinyIntType; - } else if (tableObjType == ObTableSmallIntType) { - return ObObjType.ObSmallIntType; - } else if (tableObjType == ObTableInt32Type) { - return ObObjType.ObInt32Type; - } else if (tableObjType == ObTableInt64Type) { - return ObObjType.ObInt64Type; - } else if (tableObjType == ObTableVarcharType) { - return ObObjType.ObVarcharType; - } else if (tableObjType == ObTableVarbinaryType) { - return ObObjType.ObVarcharType; - } else if (tableObjType == ObTableDoubleType) { - return ObObjType.ObDoubleType; - } else if (tableObjType == ObTableFloatType) { - return ObObjType.ObFloatType; - } else if (tableObjType == ObTableTimestampType) { - return ObObjType.ObTimestampType; - } else if (tableObjType == ObTableDateTimeType) { - return ObObjType.ObDateTimeType; - } else if (tableObjType == ObTableMinType || tableObjType == ObTableMaxType) { - return ObObjType.ObExtendType; + ObObjType objType = tableObjTypeMap.get(tableObjType); + if (objType == null) { + throw new IllegalArgumentException("cannot get ObTableObjType, invalid table obj type: " + + tableObjType.getClass().getName()); } - - throw new IllegalArgumentException("cannot get ObTableObjType, invalid table obj type: " - + tableObjType.getClass().getName()); + return objType; } /* * Value of. */ public static ObTableObjType valueOf(int value) { - return map.get(value); + return valueMap.get(value); } /* @@ -306,6 +347,14 @@ public int getEncodedSizeWithMeta(ObObj obj) { return len; } + public void decodeWithUtf8(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + objMeta.setCsType(ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI); + obj.setMeta(objMeta); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); + } + public static int DEFAULT_TABLE_OBJ_TYPE_SIZE = 1; public static int DEFAULT_TABLE_OBJ_META_SIZE = 4; } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java index 6b1b80dc..d1f1b9f8 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java @@ -160,7 +160,15 @@ public void testGet() throws Exception { `c8` double not null, `c9` timestamp(6) not null, `c10` datetime(6) not null, - `c11` int default null + `c11` int default null, + `c12` tinytext DEFAULT NULL, + `c13` text DEFAULT NULL, + `c14` mediumtext DEFAULT NULL, + `c15` longtext DEFAULT NULL, + `c16` tinyblob DEFAULT NULL, + `c17` blob DEFAULT NULL, + `c18` mediumblob DEFAULT NULL, + `c19` longblob DEFAULT NULL ); */ @Test @@ -176,13 +184,25 @@ public void testGetAllObjType() throws Exception { // prepare data Object values[][] = { - {(byte)1, (short)1, (int)1, 1L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, - {(byte)2, (short)2, (int)2, 2L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, - {(byte)3, (short)3, (int)3, 3L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, - {(byte)4, (short)4, (int)4, 4L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, - {(byte)5, (short)5, (int)5, 5L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null}, - {(byte)6, (short)6, (int)6, 6L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null} - }; + {(byte)1, (short)1, (int)1, 1L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)2, (short)2, (int)2, 2L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)3, (short)3, (int)3, 3L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)4, (short)4, (int)4, 4L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)5, (short)5, (int)5, 5L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)6, (short)6, (int)6, 6L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()} +}; int rowCnt = values.length; @@ -212,7 +232,8 @@ public void testGetAllObjType() throws Exception { for (int i = 0; i < rowCnt; i++) { Object[] curRow = values[i]; TableQuery query = query().setRowKey(row(colVal("c1", curRow[0]))) - .select("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11"); + .select("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", + "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19"); batchOperation.addOperation(query); } BatchOperationResult res = batchOperation.execute(); @@ -235,6 +256,14 @@ public void testGetAllObjType() throws Exception { { // get columns idx, 1 means get c1 int columns[][] = { + {3, 8, 14, 11, 18, 5, 12, 6, 19, 10, 15, 7, 1, 4, 9, 13, 2, 16, 17}, + {5, 15, 6, 10, 2, 12, 7, 17, 18, 11, 3, 8, 13, 1, 16, 4, 9, 14}, + {13, 10, 5, 15, 1, 8, 11, 9, 7, 16, 4, 17, 6, 2, 12, 3, 14}, + {14, 6, 9, 2, 5, 3, 15, 10, 12, 8, 11, 7, 1, 16, 4, 13}, + {11, 3, 7, 12, 8, 10, 4, 6, 15, 2, 5, 1, 14, 9, 13}, + {8, 12, 5, 14, 3, 7, 1, 11, 6, 9, 10, 2, 4, 13}, + {10, 4, 13, 1, 8, 7, 5, 11, 6, 3, 12, 2, 9}, + {9, 3, 10, 5, 7, 1, 8, 6, 11, 4, 12, 2}, {7, 3, 11, 5, 6, 2, 9, 1, 8, 4, 10}, {5, 2, 7, 9, 1, 8, 6, 3, 10, 4}, {5, 3, 1, 8, 6, 4, 7, 2, 9}, diff --git a/src/test/resources/ci.sql b/src/test/resources/ci.sql index 58a26773..6503988c 100644 --- a/src/test/resources/ci.sql +++ b/src/test/resources/ci.sql @@ -444,8 +444,17 @@ CREATE TABLE IF NOT EXISTS `test_table_object` ( `c8` double not null, `c9` timestamp(6) not null, `c10` datetime(6) not null, - `c11` int default null + `c11` int default null, + `c12` tinytext DEFAULT NULL, + `c13` text DEFAULT NULL, + `c14` mediumtext DEFAULT NULL, + `c15` longtext DEFAULT NULL, + `c16` tinyblob DEFAULT NULL, + `c17` blob DEFAULT NULL, + `c18` mediumblob DEFAULT NULL, + `c19` longblob DEFAULT NULL ); + CREATE TABLE IF NOT EXISTS `test_put` ( `id` varchar(20) NOT NULL, `c1` bigint DEFAULT NULL,