From 52480d1035f1a508c0146eee39510fbeeffaf128 Mon Sep 17 00:00:00 2001 From: Jiatao Tao <245915794@qq.com> Date: Tue, 14 May 2019 17:29:31 +0800 Subject: [PATCH] #3, reduce ByteBuffer allocation. --- parquet-arrow/pom.xml | 2 +- parquet-avro/pom.xml | 2 +- parquet-benchmarks/pom.xml | 2 +- parquet-cascading/pom.xml | 2 +- parquet-cascading3/pom.xml | 2 +- parquet-cli/pom.xml | 2 +- parquet-column/pom.xml | 2 +- parquet-common/pom.xml | 2 +- parquet-encoding/pom.xml | 2 +- parquet-format-structures/pom.xml | 2 +- parquet-generator/pom.xml | 2 +- parquet-hadoop-bundle/pom.xml | 2 +- parquet-hadoop/pom.xml | 2 +- .../org/apache/parquet/hadoop/codec/SnappyDecompressor.java | 3 ++- parquet-hive-bundle/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-0.10-binding/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-0.12-binding/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-binding-bundle/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-binding-factory/pom.xml | 2 +- .../parquet-hive-binding-interface/pom.xml | 2 +- parquet-hive/parquet-hive-binding/pom.xml | 2 +- parquet-hive/parquet-hive-storage-handler/pom.xml | 2 +- parquet-hive/pom.xml | 2 +- parquet-jackson/pom.xml | 2 +- parquet-pig-bundle/pom.xml | 2 +- parquet-pig/pom.xml | 2 +- parquet-protobuf/pom.xml | 2 +- parquet-scala/pom.xml | 2 +- parquet-scrooge/pom.xml | 2 +- parquet-thrift/pom.xml | 2 +- parquet-tools/pom.xml | 2 +- pom.xml | 2 +- 32 files changed, 33 insertions(+), 32 deletions(-) diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml index 4403431545..09867eed8e 100644 --- a/parquet-arrow/pom.xml +++ b/parquet-arrow/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml index c6459a3653..c18abb8453 100644 --- a/parquet-avro/pom.xml +++ b/parquet-avro/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml index 0694a363e4..f06e5d8229 100644 --- a/parquet-benchmarks/pom.xml +++ b/parquet-benchmarks/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-cascading/pom.xml b/parquet-cascading/pom.xml index 18e113eb35..17c3ac9cea 100644 --- a/parquet-cascading/pom.xml +++ b/parquet-cascading/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-cascading3/pom.xml b/parquet-cascading3/pom.xml index b8cc39607b..bddb478268 100644 --- a/parquet-cascading3/pom.xml +++ b/parquet-cascading3/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml index 2f29436b64..adf676c2a5 100644 --- a/parquet-cli/pom.xml +++ b/parquet-cli/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index 449329713c..f1326902cc 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml index 6658806f2d..af5b5263bd 100644 --- a/parquet-common/pom.xml +++ b/parquet-common/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml index 7f649f5870..7b50bca485 100644 --- a/parquet-encoding/pom.xml +++ b/parquet-encoding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index 5ea6ac2626..4f951dbcf4 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -24,7 +24,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 parquet-format-structures diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml index 6ece421d94..bf010e0e97 100644 --- a/parquet-generator/pom.xml +++ b/parquet-generator/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml index 1d0f5a5c41..3b231a8f89 100644 --- a/parquet-hadoop-bundle/pom.xml +++ b/parquet-hadoop-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index 7033f5ecad..eec80f1a8a 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java index c3da63f9c4..124a0a8d39 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java @@ -101,7 +101,8 @@ public synchronized void setInput(byte[] buffer, int off, int len) { SnappyUtil.validateBuffer(buffer, off, len); if (inputBuffer.capacity() - inputBuffer.position() < len) { - ByteBuffer newBuffer = ByteBuffer.allocateDirect(inputBuffer.position() + len); + int maxSize = Math.max(inputBuffer.position() * 2 , inputBuffer.position() + len); + ByteBuffer newBuffer = ByteBuffer.allocateDirect(maxSize); inputBuffer.rewind(); newBuffer.put(inputBuffer); ByteBuffer oldBuffer = inputBuffer; diff --git a/parquet-hive-bundle/pom.xml b/parquet-hive-bundle/pom.xml index a1e18be9a1..d11863567e 100644 --- a/parquet-hive-bundle/pom.xml +++ b/parquet-hive-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml index ecd7c7a153..bb20ea55b4 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml index 449011ce9e..93bb709ff8 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml index e73ffea74b..693481f364 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml @@ -23,7 +23,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml index a6546941d5..99d7cb0771 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml index 4aca68067b..35725e2f92 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/pom.xml b/parquet-hive/parquet-hive-binding/pom.xml index 7154058417..c90d1098b1 100644 --- a/parquet-hive/parquet-hive-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/parquet-hive-storage-handler/pom.xml b/parquet-hive/parquet-hive-storage-handler/pom.xml index 6e4b2c7cad..66b77e1b22 100644 --- a/parquet-hive/parquet-hive-storage-handler/pom.xml +++ b/parquet-hive/parquet-hive-storage-handler/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-hive/pom.xml b/parquet-hive/pom.xml index 31d3fb1bc0..6f6d381a44 100644 --- a/parquet-hive/pom.xml +++ b/parquet-hive/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 5843269cbc..44ede63723 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml index 76080fc3db..48bffe8ca2 100644 --- a/parquet-pig-bundle/pom.xml +++ b/parquet-pig-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml index b07a38138e..897527ed83 100644 --- a/parquet-pig/pom.xml +++ b/parquet-pig/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml index 049d4d3a05..82a2f0fc70 100644 --- a/parquet-protobuf/pom.xml +++ b/parquet-protobuf/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml index c389ddc29a..344bf475a2 100644 --- a/parquet-scala/pom.xml +++ b/parquet-scala/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml index b0a2044810..887be5c424 100644 --- a/parquet-scrooge/pom.xml +++ b/parquet-scrooge/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml index f37ee9b70d..ae78bedba1 100644 --- a/parquet-thrift/pom.xml +++ b/parquet-thrift/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/parquet-tools/pom.xml b/parquet-tools/pom.xml index c616fc2207..542c4aaa37 100644 --- a/parquet-tools/pom.xml +++ b/parquet-tools/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 4.0.0 diff --git a/pom.xml b/pom.xml index 958f86a8e7..24ed4b5a0c 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.apache.parquet parquet - 1.12.0-kylin-r1 + 1.12.0-kylin-r2 pom Apache Parquet MR