diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..6ac2a58 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,82 @@ +version: 2 + +workflows: + version: 2 + main: + jobs: + - build_2_4 + - build_2_5 + - build_2_6 +jobs: + build_2_4: + resource_class: small + docker: + - image: ruby:2.4 + - image: elastic/elasticsearch:6.8.2 + environment: + - xpack.security.enabled=false + steps: + - restore_cache: + keys: + - gem-cache-2_4-v2-{{ checksum "Gemfile.lock" }} + - gem-cache-2_4-v2- + - checkout + - run: + name: Install Ruby Dependencies + command: bundle install + - run: + name: Run Tests + command: bundle exec rspec + - save_cache: + key: gem-cache-2_4-v2-{{ checksum "Gemfile.lock" }} + paths: + - ./vendor/bundle + - ./vendor/cache + build_2_5: + resource_class: small + docker: + - image: ruby:2.5 + - image: elastic/elasticsearch:6.8.2 + environment: + - xpack.security.enabled=false + steps: + - restore_cache: + keys: + - gem-cache-2_5-v2-{{ checksum "Gemfile.lock" }} + - gem-cache-2_5-v2- + - checkout + - run: + name: Install Ruby Dependencies + command: bundle install + - run: + name: Run Tests + command: bundle exec rspec + - save_cache: + key: gem-cache-2_5-v2-{{ checksum "Gemfile.lock" }} + paths: + - ./vendor/bundle + - ./vendor/cache + build_2_6: + resource_class: small + docker: + - image: ruby:2.5 + - image: elastic/elasticsearch:6.8.2 + environment: + - xpack.security.enabled=false + steps: + - restore_cache: + keys: + - gem-cache-2_6-v2-{{ checksum "Gemfile.lock" }} + - gem-cache-2_6-v2- + - checkout + - run: + name: Install Ruby Dependencies + command: bundle install + - run: + name: Run Tests + command: bundle exec rspec + - save_cache: + key: gem-cache-2_6-v2-{{ checksum "Gemfile.lock" }} + paths: + - ./vendor/bundle + - ./vendor/cache diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e25b8e6..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: ruby -before_install: - - gem update bundler - - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-5.1.1.deb && sudo service elasticsearch restart -rvm: - - 2.2.2 - - 2.2.3 -addons: - apt: - packages: - - openjdk-8-jre-headless -before_script: - - sleep 5 -env: - global: - - secure: "HZa3D2GGwC6Jl062LJulbWZLTCieeBFC3FOJrArC5ul7ACGR5CEtANe0/UTnIf/Ad40p7I5VhiTdNFTcHunTbNc7Ae7dE5fOkiBHtxo/zwgpvHZK0iPvIoxsSfdcHobHeaF7NvfcXUkYUKcdRUyplHdB56eHQqYPVsah66K/4XA=" -sudo: true diff --git a/CHANGELOG b/CHANGELOG index 73be367..724c7fb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +v0.13.0 + - changes needed for use with Elasticsearch v7 + - handle the v7 "total" as an object rather than a scalar + - use timestamp with no colons + - update elasticsearch gem version for consistency with target ES version + - expose refresh_index to force write (because in v7, flush no longer forces writes) + - allow for optional 'include_type_name_on_create' arg so that the :include_type_name can be passed v0.12.1 - use Arel.sql to avoid unsafe sql and eliminate deprecation warnings when used in Rails projects v0.12.0 diff --git a/Gemfile b/Gemfile index 2d0fd98..9241e72 100644 --- a/Gemfile +++ b/Gemfile @@ -3,4 +3,4 @@ source 'https://rubygems.org' # Specify your gem's dependencies in elasticity.gemspec gemspec -gem "elasticsearch", "5.0.4" +gem "elasticsearch", "7.2.0" diff --git a/README.md b/README.md index ccb45ff..66a803b 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Or install it yourself as: ### Version Support This gem has [elasticsearch-ruby](https://github.com/elastic/elasticsearch-ruby) as a dependency. In order to use different versions of elasticsearch you will need to match your version of elasticsearch-ruby to the version of elasticsearch you want to use ([see here](https://github.com/elastic/elasticsearch-ruby#compatibility). Elasticity should work across all versions of elastisearch-ruby, although they have not all been tested so there are likely edge cases. -Currently tests are run on travis ci against elasticsearch 5.1.1 with elasticsearch-ruby 5.0.3. +Currently tests are run on CirlceCI against elasticsearch 6.8.2 with elasticsearch-ruby 7.2.0. ### Configuration diff --git a/elasticity.gemspec b/elasticity.gemspec index ee91940..89edd80 100644 --- a/elasticity.gemspec +++ b/elasticity.gemspec @@ -28,6 +28,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "byebug" spec.add_development_dependency "codeclimate-test-reporter" spec.add_development_dependency "redis" + spec.add_development_dependency "timecop" spec.add_dependency "activesupport", ">= 4.0.0", "< 6" spec.add_dependency "activemodel", ">= 4.0.0", "< 6" diff --git a/es-elasticity-0.12.0.pre.rc1.gem b/es-elasticity-0.12.0.pre.rc1.gem deleted file mode 100644 index ceb2c18..0000000 Binary files a/es-elasticity-0.12.0.pre.rc1.gem and /dev/null differ diff --git a/lib/elasticity/index_config.rb b/lib/elasticity/index_config.rb index cfcb173..0d2ef6a 100644 --- a/lib/elasticity/index_config.rb +++ b/lib/elasticity/index_config.rb @@ -10,7 +10,7 @@ class SubclassError < StandardError; end VERSION_FOR_SUBCLASS_ERROR = "7.0.0".freeze ATTRS = [ :index_base_name, :document_type, :mapping, :strategy, :subclasses, - :settings + :settings, :use_new_timestamp_format, :include_type_name_on_create ].freeze VALIDATABLE_ATTRS = [:index_base_name, :document_type, :strategy].freeze @@ -39,7 +39,8 @@ def client def definition return @definition if defined?(@definition) @definition = { - settings: merge_settings, mappings: { @document_type => @mapping.nil? ? {} : @mapping.deep_stringify_keys } + settings: merge_settings, + mappings: { @document_type => @mapping.nil? ? {} : @mapping.deep_stringify_keys } } subclasses.each do |doc_type, subclass| @definition[:mappings][doc_type] = subclass.constantize.mapping diff --git a/lib/elasticity/index_mapper.rb b/lib/elasticity/index_mapper.rb index 5c12014..76e7af9 100644 --- a/lib/elasticity/index_mapper.rb +++ b/lib/elasticity/index_mapper.rb @@ -11,6 +11,7 @@ def self.set_delegates(obj, to) :index_exists?, :remap!, :flush_index, + :refresh_index, :index_document, :search, :get, @@ -27,7 +28,7 @@ def self.set_delegates(obj, to) def initialize(document_klass, index_config) @document_klass = document_klass @index_config = index_config - @strategy = @index_config.strategy.new(@index_config.client, @index_config.fq_index_base_name, @index_config.document_type) + @strategy = @index_config.strategy.new(@index_config.client, @index_config.fq_index_base_name, @index_config.document_type, @index_config.use_new_timestamp_format, @index_config.include_type_name_on_create) end delegate( @@ -71,10 +72,16 @@ def remap!(retry_delete_on_recoverable_errors: true, retry_delay: 30, max_delay: end # Flushes the index, forcing any writes + # note that v7 no longer forces any writes on flush def flush_index @strategy.flush end + # Resfreshes the index, forcing any writes + def refresh_index + @strategy.refresh + end + # Index the given document def index_document(id, document_hash) @strategy.index_document(document_type, id, document_hash) diff --git a/lib/elasticity/instrumented_client.rb b/lib/elasticity/instrumented_client.rb index 66fed1e..be07fb5 100644 --- a/lib/elasticity/instrumented_client.rb +++ b/lib/elasticity/instrumented_client.rb @@ -1,6 +1,6 @@ module Elasticity class InstrumentedClient - INDICES_METHODS = %w(exists create delete get_settings get_mapping flush get_alias get_aliases put_alias delete_alias exists_alias update_aliases) + INDICES_METHODS = %w(exists create delete get_settings get_mapping flush refresh get_alias get_aliases put_alias delete_alias exists_alias update_aliases) INDEX_METHODS = %w(index delete get mget search count msearch scroll delete_by_query bulk) def initialize(client) diff --git a/lib/elasticity/search.rb b/lib/elasticity/search.rb index 48f41ac..28c3a66 100644 --- a/lib/elasticity/search.rb +++ b/lib/elasticity/search.rb @@ -87,7 +87,7 @@ def active_records(relation) class LazySearch include Enumerable - delegate :each, :size, :length, :[], :+, :-, :&, :|, :total, :per_page, + delegate :each, :size, :length, :[], :+, :-, :&, :|, :total, :per_page, :to_ary, :total_pages, :current_page, :next_page, :previous_page, :aggregations, to: :search_results attr_accessor :search_definition @@ -147,7 +147,12 @@ def blank? end def total - search["hits"]["total"] + res = search["hits"]["total"] + if res.is_a?(::Hash) + res["value"] + else + res + end end def each_batch @@ -237,7 +242,12 @@ def metadata end def total - metadata[:total] + res = metadata[:total] + if res.is_a?(::Hash) + res["value"] + else + res + end end def suggestions @@ -311,7 +321,12 @@ def aggregations end def total - @response["hits"]["total"] + res = @response["hits"]["total"] + if res.is_a?(::Hash) + res["value"] + else + res + end end alias_method :total_entries, :total diff --git a/lib/elasticity/strategies/alias_index.rb b/lib/elasticity/strategies/alias_index.rb index d0b9569..fe790b1 100644 --- a/lib/elasticity/strategies/alias_index.rb +++ b/lib/elasticity/strategies/alias_index.rb @@ -11,11 +11,15 @@ class AliasIndex STATUSES = [:missing, :ok] - def initialize(client, index_base_name, document_type) + def initialize(client, index_base_name, document_type, use_new_timestamp_format = false, include_type_name_on_create = true) @client = client @main_alias = index_base_name @update_alias = "#{index_base_name}_update" @document_type = document_type + + # included for compatibility with v7 + @use_new_timestamp_format = use_new_timestamp_format + @include_type_name_on_create = include_type_name_on_create end def ref_index_name @@ -186,11 +190,12 @@ def update_indexes def create(index_def) if missing? - index_name = create_index(index_def) + name = create_index(index_def) + @created_index_name = name @client.index_update_aliases(body: { actions: [ - { add: { index: index_name, alias: @main_alias } }, - { add: { index: index_name, alias: @update_alias } }, + { add: { index: name, alias: @main_alias } }, + { add: { index: name, alias: @update_alias } }, ] }) else @@ -257,6 +262,10 @@ def flush @client.index_flush(index: @update_alias) end + def refresh + @client.index_refresh(index: @update_alias) + end + def settings @client.index_get_settings(index: @main_alias, type: @document_type).values.first rescue Elasticsearch::Transport::Transport::Errors::NotFound @@ -279,11 +288,20 @@ def mapping private + def build_index_name + ts = String.new + if @use_new_timestamp_format == true + ts = Time.now.utc.strftime("%Y%m%d%H%M%S%6N") + else + ts = Time.now.utc.strftime("%Y-%m-%d_%H:%M:%S.%6N") + end + "#{@main_alias}-#{ts}" + end + def create_index(index_def) - ts = Time.now.utc.strftime("%Y-%m-%d_%H:%M:%S.%6N") - index_name = "#{@main_alias}-#{ts}" - @client.index_create(index: index_name, body: index_def) - index_name + name = build_index_name + @client.index_create(index: name, body: index_def, include_type_name: @include_type_name_on_create) + name end def retryable_error?(e) diff --git a/lib/elasticity/strategies/single_index.rb b/lib/elasticity/strategies/single_index.rb index f2d78ce..1f56cb6 100644 --- a/lib/elasticity/strategies/single_index.rb +++ b/lib/elasticity/strategies/single_index.rb @@ -3,10 +3,16 @@ module Strategies class SingleIndex STATUSES = [:missing, :ok] - def initialize(client, index_name, document_type) + def initialize(client, index_name, document_type, use_new_timestamp_format = false, include_type_name_on_create = true) @client = client @index_name = index_name @document_type = document_type + + # included for compatibility with v7 + @include_type_name_on_create = include_type_name_on_create + + # not currently used. included for argument compatiblity with AliasStrategy + @use_new_timestamp_format = use_new_timestamp_format end def ref_index_name @@ -23,7 +29,7 @@ def missing? def create(index_def) if missing? - @client.index_create(index: @index_name, body: index_def) + @client.index_create(index: @index_name, body: index_def, include_type_name: @include_type_name_on_create) else raise IndexError.new(@index_name, "index already exist") end @@ -101,6 +107,10 @@ def mapping def flush @client.index_flush(index: @index_name) end + + def refresh + @client.index_refresh(index: @index_name) + end end end end diff --git a/lib/elasticity/version.rb b/lib/elasticity/version.rb index 40f90b8..45117ad 100644 --- a/lib/elasticity/version.rb +++ b/lib/elasticity/version.rb @@ -1,3 +1,3 @@ module Elasticity - VERSION = "0.12.1" + VERSION = "0.13.0" end diff --git a/spec/functional/persistence_spec.rb b/spec/functional/persistence_spec.rb index 6721ddf..6490859 100644 --- a/spec/functional/persistence_spec.rb +++ b/spec/functional/persistence_spec.rb @@ -71,88 +71,6 @@ def to_document end end - describe 'multi mapping index' do - class Animal < Elasticity::Document - configure do |c| - c.index_base_name = "cats_and_dogs" - c.strategy = Elasticity::Strategies::SingleIndex - c.subclasses = { cat: "Cat", dog: "Dog" } - end - end - - class Cat < Animal - configure do |c| - c.index_base_name = "cats_and_dogs" - c.strategy = Elasticity::Strategies::SingleIndex - c.document_type = "cat" - - c.mapping = { "properties" => { - name: { type: "text", index: true }, - age: { type: "integer" } - } } - end - - attr_accessor :name, :age - - def to_document - { name: name, age: age } - end - end - - class Dog < Animal - configure do |c| - c.index_base_name = "cats_and_dogs" - c.strategy = Elasticity::Strategies::SingleIndex - c.document_type = "dog" - c.mapping = { "properties" => { - name: { type: "text", index: true }, - age: { type: "integer" }, - hungry: { type: "boolean" } - } } - end - attr_accessor :name, :age, :hungry - - def to_document - { name: name, age: age, hungry: hungry } - end - end - - before do - Animal.recreate_index - @elastic_search_client.cluster.health wait_for_status: 'yellow' - end - - it "successful index, update, search, count and deletes" do - cat = Cat.new(name: "felix", age: 10) - dog = Dog.new(name: "fido", age: 4, hungry: true) - - cat.update - dog.update - - Animal.flush_index - - results = Animal.search({}) - expect(results.total).to eq 2 - expect(results.map(&:class)).to include(Cat, Dog) - - results = Cat.search({}) - expect(results.total).to eq 1 - expect(results.first.class).to eq Cat - - results = Dog.search({}) - expect(results.total).to eq 1 - expect(results.first.class).to eq Dog - - cat.delete - Animal.flush_index - - results = Animal.search({}) - expect(results.total).to eq 1 - expect(results.map(&:class)).to include(Dog) - expect(results.scan_documents.count).to eq(1) - end - end - describe "alias index strategy" do subject do Class.new(Elasticity::Document) do diff --git a/spec/rspec_config.rb b/spec/rspec_config.rb index 2f7fbdd..4f6ca91 100644 --- a/spec/rspec_config.rb +++ b/spec/rspec_config.rb @@ -6,6 +6,7 @@ require "elasticity" require "pry" require "byebug" +require "timecop" def elastic_search_client return @elastic_search_client if defined?(@elastic_search_client) diff --git a/spec/units/index_config_spec.rb b/spec/units/index_config_spec.rb index 51b968d..8bf872a 100644 --- a/spec/units/index_config_spec.rb +++ b/spec/units/index_config_spec.rb @@ -90,6 +90,44 @@ class Multied < Elasticity::Document end end + describe "passing the time_stamp option" do + it "allows passing of a time_stamp_format option" do + config = described_class.new(elasticity_config, defaults) {} + expect(config.index_base_name).to eql('users') + expect(config.document_type).to eql('user') + expect(config.use_new_timestamp_format).to be_falsy + + config = described_class.new(elasticity_config, defaults) do |c| + c.index_base_name = 'user_documents' + c.document_type = 'users' + c.use_new_timestamp_format = true + end + + expect(config.index_base_name).to eql('user_documents') + expect(config.document_type).to eql('users') + expect(config.use_new_timestamp_format).to be_truthy + end + end + + describe "passing the include_type_name option" do + it "allows passing of a include_type_name_on_create option" do + config = described_class.new(elasticity_config, defaults) {} + expect(config.index_base_name).to eql('users') + expect(config.document_type).to eql('user') + expect(config.include_type_name_on_create).to be_falsy + + config = described_class.new(elasticity_config, defaults) do |c| + c.index_base_name = 'user_documents' + c.document_type = 'users' + c.include_type_name_on_create = true + end + + expect(config.index_base_name).to eql('user_documents') + expect(config.document_type).to eql('users') + expect(config.include_type_name_on_create).to be_truthy + end + end + def stub_version(version) allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:versions).and_return([version]) end diff --git a/spec/units/search_spec.rb b/spec/units/search_spec.rb index ec9ccad..7fcc091 100644 --- a/spec/units/search_spec.rb +++ b/spec/units/search_spec.rb @@ -13,6 +13,13 @@ ]}} end + let :full_response_v7 do + { "hits" => { "total" => { "value" => 2, "relation" => "eq" }, "hits" => [ + { "_id" => 1, "_source" => { "name" => "foo" } }, + { "_id" => 2, "_source" => { "name" => "bar" } }, + ]}} + end + let :aggregations do { "logins_count" => { "value" => 1495 }, @@ -101,6 +108,27 @@ def ==(other) expect(Array(docs)).to eq expected end + it "handles the v7 'total' response object" do + expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response_v7) + + docs = subject.documents(mapper) + expect(docs.total).to eq 2 + + expected = [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")] + + expect(docs.total).to eq 2 + expect(docs.size).to eq expected.size + + expect(docs).to_not be_empty + expect(docs).to_not be_blank + + expect(docs[0].name).to eq expected[0].name + expect(docs[1].name).to eq expected[1].name + + expect(docs.each.first).to eq expected[0] + expect(Array(docs)).to eq expected + end + it "searches and the index returns aggregations" do expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response_with_aggregations) diff --git a/spec/units/strategies/alias_index_spec.rb b/spec/units/strategies/alias_index_spec.rb new file mode 100644 index 0000000..ef0cd4a --- /dev/null +++ b/spec/units/strategies/alias_index_spec.rb @@ -0,0 +1,117 @@ +RSpec.describe Elasticity::Strategies::AliasIndex, elasticsearch: true do + subject do + described_class.new(Elasticity.config.client, "test_index_name", "document") + end + + let :index_def do + { + "mappings" => { + "document" => { + "properties" => { + "name" => { "type" => "text" } + } + } + } + } + end + + after do + subject.delete_if_defined + end + + it "allows creating, recreating and deleting an index" do + subject.create(index_def) + expect(subject.mapping).to eq(index_def) + + subject.recreate(index_def) + expect(subject.mapping).to eq(index_def) + + subject.delete + expect(subject.mapping).to be nil + end + + it "returns nil for mapping and settings when index does not exist" do + expect(subject.mapping).to be nil + expect(subject.settings).to be nil + end + + context "naming a new index" do + it "will use the oirignal timestamp format by default" do + time = Time.new(2019, 10, 11, 12, 13, 14, "+00:00") + Timecop.freeze(time) do + subject.create(index_def) + subject.index_document("document", 1, name: "test") + + doc = subject.get_document("document", 1) + expect(doc["_index"]).to eq("test_index_name-2019-10-11_12:13:14.000000") + end + end + + it "will use the new timestamp format if direcrted" do + time = Time.new(2019, 10, 11, 12, 13, 14, "+00:00") + Timecop.freeze(time) do + subject = described_class.new(Elasticity.config.client, "test_index_name", "document", true) + subject.create(index_def) + subject.index_document("document", 1, name: "test") + + doc = subject.get_document("document", 1) + expect(doc["_index"]).to eq("test_index_name-20191011121314000000") + end + end + end + + context "with existing index" do + before do + subject.create_if_undefined(index_def) + end + + it "allows adding, getting and removing documents from the index" do + subject.index_document("document", 1, name: "test") + + doc = subject.get_document("document", 1) + expect(doc["_source"]["name"]).to eq("test") + + subject.delete_document("document", 1) + expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end + + it "allows batching index and delete actions" do + results_a = subject.bulk do |b| + b.index "document", 1, name: "foo" + end + expect(results_a["errors"]).to be_falsey + + results_b = subject.bulk do |b| + b.index "document", 2, name: "bar" + b.delete "document", 1 + end + + expect(results_b["errors"]).to be_falsey + + subject.flush + + expected = { + "_type"=>"document", + "_id"=>"2", + "_version"=>1, + "found"=>true, + "_source"=>{"name"=>"bar"} + } + expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect(subject.get_document("document", 2)).to include(expected) + end + + it "allows deleting by query" do + subject.index_document("document", 1, name: "foo") + subject.index_document("document", 2, name: "bar") + + subject.flush + subject.delete_by_query("document", query: { term: { name: "foo" } }) + + expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect { subject.get_document("document", 2) }.to_not raise_error + + subject.flush + end + end +end diff --git a/spec/units/strategies/single_index_spec.rb b/spec/units/strategies/single_index_spec.rb index 860602c..9134e48 100644 --- a/spec/units/strategies/single_index_spec.rb +++ b/spec/units/strategies/single_index_spec.rb @@ -65,8 +65,16 @@ subject.flush + expected = { + "_index"=>"test_index_name", + "_type"=>"document", + "_id"=>"2", + "_version"=>1, + "found"=>true, + "_source"=>{"name"=>"bar"} + } expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) - expect(subject.get_document("document", 2)).to eq({"_index"=>"test_index_name", "_type"=>"document", "_id"=>"2", "_version"=>1, "found"=>true, "_source"=>{"name"=>"bar"}}) + expect(subject.get_document("document", 2)).to include(expected) end it "allows deleting by query" do