From 52480d1035f1a508c0146eee39510fbeeffaf128 Mon Sep 17 00:00:00 2001
From: Jiatao Tao <245915794@qq.com>
Date: Tue, 14 May 2019 17:29:31 +0800
Subject: [PATCH] #3, reduce ByteBuffer allocation.
---
parquet-arrow/pom.xml | 2 +-
parquet-avro/pom.xml | 2 +-
parquet-benchmarks/pom.xml | 2 +-
parquet-cascading/pom.xml | 2 +-
parquet-cascading3/pom.xml | 2 +-
parquet-cli/pom.xml | 2 +-
parquet-column/pom.xml | 2 +-
parquet-common/pom.xml | 2 +-
parquet-encoding/pom.xml | 2 +-
parquet-format-structures/pom.xml | 2 +-
parquet-generator/pom.xml | 2 +-
parquet-hadoop-bundle/pom.xml | 2 +-
parquet-hadoop/pom.xml | 2 +-
.../org/apache/parquet/hadoop/codec/SnappyDecompressor.java | 3 ++-
parquet-hive-bundle/pom.xml | 2 +-
.../parquet-hive-binding/parquet-hive-0.10-binding/pom.xml | 2 +-
.../parquet-hive-binding/parquet-hive-0.12-binding/pom.xml | 2 +-
.../parquet-hive-binding/parquet-hive-binding-bundle/pom.xml | 2 +-
.../parquet-hive-binding/parquet-hive-binding-factory/pom.xml | 2 +-
.../parquet-hive-binding-interface/pom.xml | 2 +-
parquet-hive/parquet-hive-binding/pom.xml | 2 +-
parquet-hive/parquet-hive-storage-handler/pom.xml | 2 +-
parquet-hive/pom.xml | 2 +-
parquet-jackson/pom.xml | 2 +-
parquet-pig-bundle/pom.xml | 2 +-
parquet-pig/pom.xml | 2 +-
parquet-protobuf/pom.xml | 2 +-
parquet-scala/pom.xml | 2 +-
parquet-scrooge/pom.xml | 2 +-
parquet-thrift/pom.xml | 2 +-
parquet-tools/pom.xml | 2 +-
pom.xml | 2 +-
32 files changed, 33 insertions(+), 32 deletions(-)
diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml
index 4403431545..09867eed8e 100644
--- a/parquet-arrow/pom.xml
+++ b/parquet-arrow/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml
index c6459a3653..c18abb8453 100644
--- a/parquet-avro/pom.xml
+++ b/parquet-avro/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml
index 0694a363e4..f06e5d8229 100644
--- a/parquet-benchmarks/pom.xml
+++ b/parquet-benchmarks/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-cascading/pom.xml b/parquet-cascading/pom.xml
index 18e113eb35..17c3ac9cea 100644
--- a/parquet-cascading/pom.xml
+++ b/parquet-cascading/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-cascading3/pom.xml b/parquet-cascading3/pom.xml
index b8cc39607b..bddb478268 100644
--- a/parquet-cascading3/pom.xml
+++ b/parquet-cascading3/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
index 2f29436b64..adf676c2a5 100644
--- a/parquet-cli/pom.xml
+++ b/parquet-cli/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml
index 449329713c..f1326902cc 100644
--- a/parquet-column/pom.xml
+++ b/parquet-column/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
index 6658806f2d..af5b5263bd 100644
--- a/parquet-common/pom.xml
+++ b/parquet-common/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml
index 7f649f5870..7b50bca485 100644
--- a/parquet-encoding/pom.xml
+++ b/parquet-encoding/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml
index 5ea6ac2626..4f951dbcf4 100644
--- a/parquet-format-structures/pom.xml
+++ b/parquet-format-structures/pom.xml
@@ -24,7 +24,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
parquet-format-structures
diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
index 6ece421d94..bf010e0e97 100644
--- a/parquet-generator/pom.xml
+++ b/parquet-generator/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
index 1d0f5a5c41..3b231a8f89 100644
--- a/parquet-hadoop-bundle/pom.xml
+++ b/parquet-hadoop-bundle/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
index 7033f5ecad..eec80f1a8a 100644
--- a/parquet-hadoop/pom.xml
+++ b/parquet-hadoop/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java
index c3da63f9c4..124a0a8d39 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/SnappyDecompressor.java
@@ -101,7 +101,8 @@ public synchronized void setInput(byte[] buffer, int off, int len) {
SnappyUtil.validateBuffer(buffer, off, len);
if (inputBuffer.capacity() - inputBuffer.position() < len) {
- ByteBuffer newBuffer = ByteBuffer.allocateDirect(inputBuffer.position() + len);
+ int maxSize = Math.max(inputBuffer.position() * 2 , inputBuffer.position() + len);
+ ByteBuffer newBuffer = ByteBuffer.allocateDirect(maxSize);
inputBuffer.rewind();
newBuffer.put(inputBuffer);
ByteBuffer oldBuffer = inputBuffer;
diff --git a/parquet-hive-bundle/pom.xml b/parquet-hive-bundle/pom.xml
index a1e18be9a1..d11863567e 100644
--- a/parquet-hive-bundle/pom.xml
+++ b/parquet-hive-bundle/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
index ecd7c7a153..bb20ea55b4 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive-binding
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
index 449011ce9e..93bb709ff8 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive-binding
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml
index e73ffea74b..693481f364 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml
@@ -23,7 +23,7 @@
org.apache.parquet
parquet-hive-binding
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
index a6546941d5..99d7cb0771 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive-binding
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
index 4aca68067b..35725e2f92 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive-binding
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-binding/pom.xml b/parquet-hive/parquet-hive-binding/pom.xml
index 7154058417..c90d1098b1 100644
--- a/parquet-hive/parquet-hive-binding/pom.xml
+++ b/parquet-hive/parquet-hive-binding/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/parquet-hive-storage-handler/pom.xml b/parquet-hive/parquet-hive-storage-handler/pom.xml
index 6e4b2c7cad..66b77e1b22 100644
--- a/parquet-hive/parquet-hive-storage-handler/pom.xml
+++ b/parquet-hive/parquet-hive-storage-handler/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet-hive
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-hive/pom.xml b/parquet-hive/pom.xml
index 31d3fb1bc0..6f6d381a44 100644
--- a/parquet-hive/pom.xml
+++ b/parquet-hive/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml
index 5843269cbc..44ede63723 100644
--- a/parquet-jackson/pom.xml
+++ b/parquet-jackson/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml
index 76080fc3db..48bffe8ca2 100644
--- a/parquet-pig-bundle/pom.xml
+++ b/parquet-pig-bundle/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml
index b07a38138e..897527ed83 100644
--- a/parquet-pig/pom.xml
+++ b/parquet-pig/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml
index 049d4d3a05..82a2f0fc70 100644
--- a/parquet-protobuf/pom.xml
+++ b/parquet-protobuf/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml
index c389ddc29a..344bf475a2 100644
--- a/parquet-scala/pom.xml
+++ b/parquet-scala/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml
index b0a2044810..887be5c424 100644
--- a/parquet-scrooge/pom.xml
+++ b/parquet-scrooge/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml
index f37ee9b70d..ae78bedba1 100644
--- a/parquet-thrift/pom.xml
+++ b/parquet-thrift/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/parquet-tools/pom.xml b/parquet-tools/pom.xml
index c616fc2207..542c4aaa37 100644
--- a/parquet-tools/pom.xml
+++ b/parquet-tools/pom.xml
@@ -21,7 +21,7 @@
org.apache.parquet
parquet
../pom.xml
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
4.0.0
diff --git a/pom.xml b/pom.xml
index 958f86a8e7..24ed4b5a0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -9,7 +9,7 @@
org.apache.parquet
parquet
- 1.12.0-kylin-r1
+ 1.12.0-kylin-r2
pom
Apache Parquet MR