From 36ec0d04b21addf67cf953be595a5d13d6f0362b Mon Sep 17 00:00:00 2001
From: Joseph Blomstedt <joe@basho.com>
Date: Thu, 28 Aug 2014 12:36:02 -0700
Subject: [PATCH] Make synctree LevelDB options user-configurable

This commit makes the LevelDB options used for ensemble synctrees
configurable in app.config.

This commit also uses the randomized write_buffer pattern used elsewhere
in Riak, where the LevelDB write buffer size is set to be a random value
within a given min/max range. Randomizing the write buffer per LevelDB
instance helps protect against multiple instances needing to flush their
write buffers at the same time.
---
 src/riak_ensemble_config.erl | 21 +++++++++++++++++++++
 src/synctree_leveldb.erl     |  6 +-----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/riak_ensemble_config.erl b/src/riak_ensemble_config.erl
index ee21bef..355c0d0 100644
--- a/src/riak_ensemble_config.erl
+++ b/src/riak_ensemble_config.erl
@@ -113,6 +113,27 @@ tree_validation() ->
 synchronous_tree_updates() ->
     get_env(synchronous_tree_updates, false).
 
+%% @doc LevelDB options used for synctree LevelDB instances.
+synctree_leveldb_opts() ->
+    DefaultWriteBufferMin = 4 * 1024 * 1024,
+    DefaultWriteBufferMax = 14 * 1024 * 1024,
+    Config = get_env(synctree_leveldb_opts,
+                     [{write_buffer_size_min, DefaultWriteBufferMin},
+                      {write_buffer_size_max, DefaultWriteBufferMax}]),
+    %% Use a variable write buffer size to prevent against all buffers being
+    %% flushed to disk at once when under a heavy uniform load.
+    WriteBufferMin = proplists:get_value(write_buffer_size_min, Config, DefaultWriteBufferMin),
+    WriteBufferMax = proplists:get_value(write_buffer_size_max, Config, DefaultWriteBufferMax),
+    {Offset, _} = random:uniform_s(1 + WriteBufferMax - WriteBufferMin, now()),
+    WriteBufferSize = WriteBufferMin + Offset,
+    Config2 = orddict:store(write_buffer_size, WriteBufferSize, Config),
+    Config3 = orddict:erase(write_buffer_size_min, Config2),
+    Config4 = orddict:erase(write_buffer_size_max, Config3),
+    Config5 = orddict:store(is_internal_db, true, Config4),
+    Config6 = orddict:store(use_bloomfilter, true, Config5),
+    Options = orddict:store(create_if_missing, true, Config6),
+    Options.
+
 get_env(Key, Default) ->
     case application:get_env(riak_ensemble, Key) of
         undefined ->
diff --git a/src/synctree_leveldb.erl b/src/synctree_leveldb.erl
index 046f218..ee8c064 100644
--- a/src/synctree_leveldb.erl
+++ b/src/synctree_leveldb.erl
@@ -174,8 +174,4 @@ timestamp({Mega, Secs, Micro}) ->
     Mega*1000*1000*1000*1000 + Secs * 1000 * 1000 + Micro.
 
 leveldb_opts() ->
-    [{is_internal_db, true},
-     {write_buffer_size, 4 * 1024 * 1024},
-     {use_bloomfilter, true},
-     {create_if_missing, true}].
-
+    riak_ensemble_config:synctree_leveldb_opts().