0.1.0

.gitignore

@@ -0,0 +1,7 @@
+Gemfile.lock
+log
+/config.yml
+.*
+!.git
+.elasticbeanstalk/
+docker.id

Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:trusty
+
+RUN apt-get update
+
+# Install ruby-2.1.1
+RUN apt-get build-dep -y ruby2.0
+RUN apt-get install -y git curl
+
+RUN git clone https://github.com/sstephenson/ruby-build.git /ruby-build
+
+ENV PATH /ruby-build/bin:$PATH
+# Patch for readline because of breaked compatibility at current version
+RUN curl -fsSL https://gist.github.com/mislav/a18b9d7f0dc5b9efc162.txt | ruby-build --patch 2.1.1 /usr/local
+RUN gem install bundler
+
+# Install kusabana
+RUN mkdir /kusabana
+WORKDIR /kusabana
+
+# Add Gemfile and run bundle install
+RUN apt-get install -y libsasl2-dev
+ADD ./Gemfile /kusabana/
+RUN bundle install
+
+# Add any other file
+RUN mv Gemfile.lock Gemfile.lock.tmp
+ADD . /kusabana/
+RUN mv Gemfile.lock.tmp Gemfile.lock
+
+EXPOSE 9292
+ENTRYPOINT ["bundle", "exec", "rake"]
+CMD ["start"]

Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 Satoshi Amemiya and VOYAGE GROUP. All rights reserved.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Django nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -0,0 +1,2 @@
+install:
+	bundle install

README.md

@@ -0,0 +1,33 @@
+kusabana
+========
+
+test proxy for Elasticsearch
+
+### Require
+
+memcached  
+ruby => 2.1.1 (maybe work also in ruby => 2.0)
+
+### Installation
+
+    git clone https://github.com/voyagegroup/kusabana.git
+    cd kusabana
+    bundle install
+
+More documents are available at http://voyagegroup.github.io/kusabana/
+
+### Contributing
+
+Patches welcome! welcome!  
+
+#### Todo
+1. Fork repo
+2. Make it change
+3. Run test (`bundle exec rake test`)
+4. Add a test for change
+5. Submit pull request
+
+## Copyright
+* Copyright (c) 2014 Satoshi Amemiya and VOYAGE GROUP. All rights reserved.
+* License
+  - 3-clause BSD license

Rakefile

@@ -0,0 +1,134 @@
+require 'bundler/gem_tasks'
+require 'elasticsearch'
+require 'yaml'
+
+config = begin
+           YAML.load_file('config.yml')
+         rescue Errno::ENOENT => e
+           YAML.load_file('./spec/config.yml')
+         end
+
+def get_pid(config)
+  begin
+    open(config['proxy']['pid'] || 'kusabana.pid').read
+  rescue Errno::ENOENT
+    nil
+  end
+end
+
+def alive?(pid)
+  begin
+    Process.kill(0, pid)
+  rescue Errno::ESRCH
+    nil
+  end
+end
+
+desc 'Start Kusabana'
+task :start do
+  if config['proxy']['daemonize']
+    pid = get_pid(config)
+    if pid && alive?(pid.to_i)
+      fail('Kusabana is already running')
+    end
+  end
+  sh 'bundle exec bin/kusabana'
+end
+
+desc 'Stop Kusabana running as a daemon'
+task :stop do
+  pid = get_pid(config)
+  if pid
+    if alive?(pid.to_i)
+      sh "kill #{pid}"
+      sleep 1
+      sh "kill -9 #{pid}" if alive?(pid.to_i)
+    end
+  end
+end
+
+desc 'Restart Kusabana running as a daemon'
+task :restart => [:stop, :start]
+
+desc 'Run RSpec'
+task :test do
+  sh 'bundle exec rspec'
+end
+
+# Tasks for Docker
+namespace :docker do
+  def get_container_id
+    begin
+      open('docker.id').read
+    rescue Errno::ENOENT
+      nil
+    end
+  end
+
+  def container_alive?(id)
+    alives = `docker ps -a -q`
+    alives.each_line do |alive|
+      return true if id.index(alive[0..-2]) == 0
+    end
+    return false
+  end
+
+  desc 'Build Docker image'
+  task :build do
+    sh 'docker build -t kusabana .'
+  end
+
+  desc 'Run Docker container'
+  task :run => 'docker:build' do
+    sh 'docker run -p 9292:9292 -t -i --rm kusabana'
+  end
+
+  desc 'Start Docker container'
+  task :start => 'docker:build' do
+    if last_id = get_container_id
+      fail('Container is already running') if container_alive?(last_id)
+    end
+    sh 'docker run -p 9292:9292 -d kusabana > docker.id'
+  end
+
+  desc 'Stop Docker container running as a daemon'
+  task :stop do
+    if last_id = get_container_id
+      sh "docker rm -f #{last_id}" if container_alive?(last_id)
+    else
+      fail('Container isn\'t running')
+   end
+  end
+
+  desc 'Restart Docker container running as a daemon'
+  task :restart => [:stop, :start]
+
+  desc 'Run RSpec within Docker container'
+  task :test => 'docker:build' do
+    sh 'docker run -p 9292:9292 --rm kusabana test'
+  end
+end
+
+namespace :dashboard do
+  desc 'Create dashboard into output elasticsearch'
+  task :create do
+    hosts = config['es']['output']['hosts'].map do |v|
+      v.inject({}) {|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+    es = Elasticsearch::Client.new(hosts: hosts)
+    body = open('kibana-dashboard.json').read
+    es.index(index: 'kibana-int', type: 'dashboard', id: 'kusabana', body: body)
+  end
+end
+
+namespace :template do
+  desc 'Create index template into output elasticsearch'
+  task :create do
+    hosts = config['es']['output']['hosts'].map do |v|
+      v.inject({}) {|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+    es = Elasticsearch::Client.new(hosts: hosts)
+    body = open('kusabana_log_template.json').read
+    es.index(index: '_template', type: 'kusabana-template-1', body: body)
+  end
+end

bin/kusabana

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# coding: UTF-8
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'kusabana'
+require 'yaml'
+
+config = YAML.load_file("config.yml")
+
+rules = []
+search_caching = Kusabana::Rule.new('POST', /^\/.+\/_search(\?.+)?$/, 300)
+timestamp_modifier = Kusabana::QueryModifier.new(/@timestamp/) do |query|
+  if query.key?('from') && query.key?('to')
+    query['from'] = query['from'] / 100000 * 100000
+    query['to'] = query['to'] / 100000 * 100000
+  end
+  query
+end
+search_caching.add_modifier(timestamp_modifier)
+rules << search_caching
+
+rules << Kusabana::Rule.new('GET', /^\/_nodes$/, 300)
+rules << Kusabana::Rule.new('GET', /^\/\S+\/_mapping$/, 300)
+rules << Kusabana::Rule.new('GET', /^\/\S+\/_aliases(\?.+)?$/, 300)
+
+Kusabana::Proxy.new(rules, config).start

config.yml.sample

@@ -0,0 +1,18 @@
+proxy:
+  host: '0.0.0.0'
+  port: 9292
+  daemonize: false
+  timeout: 15
+# output: 'log/kusabana.log' 
+# pid: 'log/kusabana.pid'
+es:
+  remote:
+    host: 'localhost'
+    port: 9200
+# output:
+#   index: 'kusabana-log-1'
+#   hosts:
+#   - host: 'localhost'
+#     port: 9200
+cache:
+  url: 'localhost:11211'

example/td-agent.conf

@@ -0,0 +1,18 @@
+<source>
+  type tail
+  path /var/log/kusabana.log
+  pos_file /var/log/td-agent/kusabana.log.pos
+  format ltsv
+  types took:float
+  time_format %Y-%m-%d %H:%M:%S %Z
+  tag kusabana.log
+</source>
+
+<match kusabana.log>
+  type elasticsearch
+  logstash_format true
+  type_name kusabana
+  host localhost
+  port 9200
+  flush_interval 10s
+</match>

kibana-dashboard.json

@@ -0,0 +1 @@
+{"user":"guest","group":"guest","title":"kusabana","dashboard":"{\"title\":\"kusabana\",\"services\":{\"query\":{\"list\":{\"0\":{\"id\":0,\"color\":\"#7EB26D\",\"alias\":\"\",\"pin\":false,\"type\":\"lucene\",\"enable\":true,\"query\":\"_type: \\\"req\\\"\"},\"1\":{\"id\":1,\"color\":\"#EAB839\",\"alias\":\"\",\"pin\":false,\"type\":\"lucene\",\"enable\":true,\"query\":\"_type: \\\"res\\\"\"},\"2\":{\"id\":2,\"color\":\"#6ED0E0\",\"alias\":\"\",\"pin\":false,\"type\":\"lucene\",\"enable\":true,\"query\":\"_type: \\\"stat\\\"\"}},\"ids\":[0,1,2]},\"filter\":{\"list\":{\"0\":{\"type\":\"time\",\"field\":\"@timestamp\",\"from\":\"now-15m\",\"to\":\"now\",\"mandate\":\"must\",\"active\":true,\"alias\":\"\",\"id\":0}},\"ids\":[0]}},\"rows\":[{\"title\":\"Graph\",\"height\":\"600px\",\"editable\":true,\"collapse\":false,\"collapsable\":true,\"panels\":[{\"span\":6,\"editable\":true,\"type\":\"histogram\",\"loadingEditor\":false,\"mode\":\"count\",\"time_field\":\"@timestamp\",\"value_field\":null,\"x-axis\":true,\"y-axis\":true,\"scale\":1,\"y_format\":\"none\",\"grid\":{\"max\":null,\"min\":null},\"queries\":{\"mode\":\"all\",\"ids\":[0,1,2]},\"annotate\":{\"enable\":false,\"query\":\"*\",\"size\":20,\"field\":\"_type\",\"sort\":[\"_score\",\"desc\"]},\"auto_int\":true,\"resolution\":100,\"interval\":\"10s\",\"intervals\":[\"auto\",\"1s\",\"1m\",\"5m\",\"10m\",\"30m\",\"1h\",\"3h\",\"12h\",\"1d\",\"1w\",\"1y\"],\"lines\":false,\"fill\":0,\"linewidth\":3,\"points\":false,\"pointradius\":5,\"bars\":true,\"stack\":true,\"spyable\":true,\"zoomlinks\":true,\"options\":true,\"legend\":true,\"show_query\":true,\"interactive\":true,\"legend_counts\":true,\"timezone\":\"browser\",\"percentage\":false,\"zerofill\":false,\"derivative\":false,\"tooltip\":{\"value_type\":\"cumulative\",\"query_as_alias\":true},\"title\":\"Histgrom\",\"scaleSeconds\":false},{\"error\":false,\"span\":6,\"editable\":true,\"type\":\"table\",\"loadingEditor\":false,\"size\":10,\"pages\":5,\"offset\":0,\"sort\":[\"efficiency\",\"desc\"],\"overflow\":\"min-height\",\"fields\":[\"count\",\"key\",\"efficiency\",\"expire\",\"from\",\"to\",\"avg\",\"min\",\"max\",\"sum\",\"@timestamp\"],\"highlight\":[],\"sortable\":true,\"header\":true,\"paging\":true,\"field_list\":false,\"all_fields\":false,\"trimFactor\":300,\"localTime\":false,\"timeField\":\"@timestamp\",\"spyable\":true,\"queries\":{\"mode\":\"selected\",\"ids\":[2]},\"style\":{\"font-size\":\"9pt\"},\"normTimes\":true,\"title\":\"Stat\"}],\"notice\":false},{\"title\":\"Table\",\"height\":\"150px\",\"editable\":true,\"collapse\":false,\"collapsable\":true,\"panels\":[{\"error\":false,\"span\":6,\"editable\":true,\"type\":\"table\",\"loadingEditor\":false,\"size\":100,\"pages\":5,\"offset\":0,\"sort\":[\"@timestamp\",\"desc\"],\"overflow\":\"min-height\",\"fields\":[\"method\",\"path\",\"@timestamp\",\"took\",\"cache\",\"key\",\"expire\",\"session\"],\"highlight\":[],\"sortable\":true,\"header\":true,\"paging\":true,\"field_list\":false,\"all_fields\":false,\"trimFactor\":300,\"localTime\":false,\"timeField\":\"@timestamp\",\"spyable\":true,\"queries\":{\"mode\":\"selected\",\"ids\":[1]},\"style\":{\"font-size\":\"9pt\"},\"normTimes\":true,\"title\":\"Response\"},{\"error\":false,\"span\":6,\"editable\":true,\"type\":\"table\",\"loadingEditor\":false,\"size\":100,\"pages\":5,\"offset\":0,\"sort\":[\"@timestamp\",\"desc\"],\"overflow\":\"min-height\",\"fields\":[\"method\",\"path\",\"@timestamp\",\"match\",\"session\"],\"highlight\":[],\"sortable\":true,\"header\":true,\"paging\":true,\"field_list\":false,\"all_fields\":false,\"trimFactor\":300,\"localTime\":false,\"timeField\":\"@timestamp\",\"spyable\":true,\"queries\":{\"mode\":\"selected\",\"ids\":[0]},\"style\":{\"font-size\":\"9pt\"},\"normTimes\":true,\"title\":\"Request\"}],\"notice\":false}],\"editable\":true,\"failover\":false,\"index\":{\"interval\":\"none\",\"pattern\":\"[logstash-]YYYY.MM.DD\",\"default\":\"kusabana-log-1\",\"warm_fields\":false},\"style\":\"dark\",\"panel_hints\":true,\"pulldowns\":[{\"type\":\"query\",\"collapse\":false,\"notice\":false,\"enable\":true,\"query\":\"*\",\"pinned\":true,\"history\":[\"_type: \\\"stat\\\"\",\"_type: \\\"res\\\"\",\"_type: \\\"req\\\"\",\"*\",\"\\\"_type\\\":\\\"res\\\"\",\"{_type:'res'}\",\"_type:'res'\",\"_type: res\"],\"remember\":10},{\"type\":\"filtering\",\"collapse\":false,\"notice\":true,\"enable\":true}],\"nav\":[{\"type\":\"timepicker\",\"collapse\":false,\"notice\":false,\"enable\":true,\"status\":\"Stable\",\"time_options\":[\"5m\",\"15m\",\"1h\",\"6h\",\"12h\",\"24h\",\"2d\",\"7d\",\"30d\"],\"refresh_intervals\":[\"5s\",\"10s\",\"30s\",\"1m\",\"5m\",\"15m\",\"30m\",\"1h\",\"2h\",\"1d\"],\"timefield\":\"@timestamp\",\"now\":true,\"filter_id\":0}],\"loader\":{\"save_gist\":false,\"save_elasticsearch\":true,\"save_local\":true,\"save_default\":true,\"save_temp\":true,\"save_temp_ttl_enable\":true,\"save_temp_ttl\":\"30d\",\"load_gist\":false,\"load_elasticsearch\":true,\"load_elasticsearch_size\":20,\"load_local\":false,\"hide\":false},\"refresh\":\"1m\"}"}

kusabana.gemspec

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+require File.expand_path('../lib/kusabana/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = "kusabana"
+  s.platform = Gem::Platform::RUBY
+  s.authors = ["Satoshi Amemiya"]
+  s.email = ["[email protected]"]
+  s.homepage = "http://voyagegroup.github.io/kusabana/"
+  s.summary = %q{Kusabana is Proxy server with caching between Kibana and ElasticSearch}
+  s.description = s.summary
+  s.version = Kusabana::VERSION
+
+  s.add_dependency 'rake', '~> 10.3'
+  s.add_dependency 'memcached', '~> 1.7'
+  s.add_dependency 'em-proxy', '~> 0.1'
+  s.add_dependency 'oj', '~> 2.9'
+  s.add_dependency 'http_parser.rb', '~> 0.6'
+  s.add_dependency 'uuid', '~> 2.3'
+  s.add_dependency 'elasticsearch', '~> 1.0'
+
+  s.add_development_dependency 'rspec', '~> 2.14'
+  s.add_development_dependency 'webmock', '~> 1.17'
+
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end