diff --git a/.gitignore b/.gitignore index a8d69ea..6937a88 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ mkmf.log /.rvmrc /.idea/ gemfiles/ -internal/log \ No newline at end of file +spec/internal/log diff --git a/.rspec b/.rspec index 19ddf02..df686f0 100644 --- a/.rspec +++ b/.rspec @@ -2,4 +2,5 @@ --tty --format progress --order random ---backtrace \ No newline at end of file +--backtrace +--require ./spec/spec_helper diff --git a/Appraisals b/Appraisals index 8f295be..0405eb2 100644 --- a/Appraisals +++ b/Appraisals @@ -5,7 +5,3 @@ end if RUBY_VERSION < '2' appraise 'activesupport3.2' do gem 'activesupport', '~> 3.2.0' end - -appraise 'activesupport4.0' do - gem 'activesupport', '~> 4.0.13' -end diff --git a/Gemfile b/Gemfile index e336dc9..646240b 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,12 @@ source 'https://rubygems.org' if RUBY_VERSION < '2' gem 'mime-types', '< 3.0' gem 'json', '< 2' + gem 'pry-debugger' + gem 'pg', '<= 0.18.4' + gem 'shoulda-matchers', '< 3.0.0' +else + gem 'pry-byebug' + gem 'test-unit' end gemspec diff --git a/app/jobs/treasury/base_job.rb b/app/jobs/treasury/base_job.rb new file mode 100644 index 0000000..c4244c9 --- /dev/null +++ b/app/jobs/treasury/base_job.rb @@ -0,0 +1,6 @@ +module Treasury + class BaseJob < Treasury::BgExecutor::Job::Indicated + acts_as_no_cancel + acts_as_critical notify_email: Treasury.configuration.job_error_notifications + end +end diff --git a/app/jobs/treasury/delayed_increment_job.rb b/app/jobs/treasury/delayed_increment_job.rb new file mode 100644 index 0000000..d6ba38f --- /dev/null +++ b/app/jobs/treasury/delayed_increment_job.rb @@ -0,0 +1,30 @@ +# coding: utf-8 + +module Treasury + class DelayedIncrementJob + include Resque::Integration + + queue :base + retrys + + # Public: Отложенный инкремент поля + # + # params - Hash: + # 'object' - Integer идентификатор + # 'field_name' - String название поля + # 'field_class' - String класс поля + # 'by' - Integer приращение + # + # Returns nothing + def self.perform(params) + object = params.fetch('object') + field_name = params.fetch('field_name') + increment = params.fetch('by') + + field = Treasury::Fields::Base.create_by_class(params.fetch('field_class')) + new_value = field.raw_value(object, field_name).to_i + increment + + field.write_data({object => {field_name => new_value}}, true) + end + end +end diff --git a/app/jobs/treasury/initialize_field_job.rb b/app/jobs/treasury/initialize_field_job.rb new file mode 100644 index 0000000..bf4de3e --- /dev/null +++ b/app/jobs/treasury/initialize_field_job.rb @@ -0,0 +1,14 @@ +module Treasury + class InitializeFieldJob < BaseJob + acts_as_singleton [:field_class] + + def execute + field = Treasury::Fields::Base.create_by_class(params[:field_class]) + field.initialize! + end + + def title + "#{self.class.name.underscore.gsub('_job', '')}: #{params[:field_class]}" + end + end +end diff --git a/app/jobs/treasury/supervisor_job.rb b/app/jobs/treasury/supervisor_job.rb new file mode 100644 index 0000000..af6f8a2 --- /dev/null +++ b/app/jobs/treasury/supervisor_job.rb @@ -0,0 +1,9 @@ +module Treasury + class SupervisorJob < BaseJob + acts_as_singleton + + def execute + Treasury::Supervisor.run + end + end +end diff --git a/app/jobs/treasury/worker_job.rb b/app/jobs/treasury/worker_job.rb new file mode 100644 index 0000000..01156c8 --- /dev/null +++ b/app/jobs/treasury/worker_job.rb @@ -0,0 +1,19 @@ +module Treasury + class WorkerJob < BaseJob + acts_as_singleton [:worker_id] + + def execute + Worker.run(params[:worker_id]) + end + + def title + "#{self.class.name.underscore.gsub('_job', '')}:#{worker.name}" + end + + protected + + def worker + @worker ||= Treasury::Models::Worker.find(params[:worker_id]) + end + end +end diff --git a/app/models/treasury/models/events_log.rb b/app/models/treasury/models/events_log.rb new file mode 100644 index 0000000..dc92045 --- /dev/null +++ b/app/models/treasury/models/events_log.rb @@ -0,0 +1,12 @@ +# coding: utf-8 +module Treasury + module Models + class EventsLog < ActiveRecord::Base + self.table_name = 'denormalization.events_log' + + def self.clear(date) + delete_all(['processed_at::date = ?', date]) + end + end + end +end diff --git a/app/models/treasury/models/field.rb b/app/models/treasury/models/field.rb new file mode 100644 index 0000000..8ed37a0 --- /dev/null +++ b/app/models/treasury/models/field.rb @@ -0,0 +1,45 @@ +# coding: utf-8 +module Treasury + module Models + class Field < ActiveRecord::Base + self.table_name = 'denormalization.fields' + self.primary_key = 'id' + + has_many :processors, + class_name: 'Treasury::Models::Processor', + dependent: :destroy, + order: 'processors.oid', + inverse_of: :field + + belongs_to :worker, class_name: 'Treasury::Models::Worker' + + scope :active, -> { where(active: true) } + scope :ordered, -> { order(arel_table[:oid]) } + scope :initialized, -> { where(state: Fields::STATE_INITIALIZED) } + scope :in_initialize, -> { where(state: Fields::STATE_IN_INITIALIZE) } + scope :for_initialize_or_in_initialize, -> do + where(state: [Fields::STATE_NEED_INITIALIZE, Fields::STATE_IN_INITIALIZE]) + end + scope :for_processing, -> { active.initialized.ordered } + + serialize :params, Hash + serialize :storage, Array + + def need_initialize! + update_attribute(:state, Fields::STATE_NEED_INITIALIZE) + end + + def need_initialize? + state == Fields::STATE_NEED_INITIALIZE + end + + def suspend + update_attribute(:active, false) + end + + def resume + update_attribute(:active, true) + end + end + end +end diff --git a/app/models/treasury/models/processor.rb b/app/models/treasury/models/processor.rb new file mode 100644 index 0000000..b457e33 --- /dev/null +++ b/app/models/treasury/models/processor.rb @@ -0,0 +1,35 @@ +# coding: utf-8 +module Treasury + module Models + class Processor < ActiveRecord::Base + self.table_name = 'denormalization.processors' + self.primary_key = 'id' + + belongs_to :queue, class_name: 'Treasury::Models::Queue', inverse_of: :processors + belongs_to :field, class_name: 'Treasury::Models::Field', inverse_of: :processors + + before_destroy :unregister_consumer + + serialize :params, Hash + + require 'treasury/pgq' + + def subscribe! + unregister_consumer + create_queue_if_needet + # TODO: check and create or enable trigger if needet! + ActiveRecord::Base.pgq_register_consumer(queue.pgq_queue_name, consumer_name, queue.work_connection) + end + + def unregister_consumer + ActiveRecord::Base.pgq_unregister_consumer(queue.pgq_queue_name, consumer_name, queue.work_connection) + end + + def create_queue_if_needet + return if queue.pgq_queue_exists? + + queue.create_pgq_queue + end + end + end +end diff --git a/app/models/treasury/models/queue.rb b/app/models/treasury/models/queue.rb new file mode 100644 index 0000000..2214391 --- /dev/null +++ b/app/models/treasury/models/queue.rb @@ -0,0 +1,175 @@ +# coding: utf-8 +module Treasury + module Models + class Queue < ActiveRecord::Base + self.table_name = 'denormalization.queues' + self.primary_key = 'id' + + TRIGGER_PREFIX = 'tr_denorm'.freeze + + has_many :processors, :class_name => 'Treasury::Models::Processor', :dependent => :destroy + + before_destroy :destroy_pgq_queue + after_create :create_pgq_queue + # + пересоздавать очередь и триггер при изменениях + + def self.generate_trigger(options = {}) + options = {:backup => true}.merge(options) + + raise ArgumentError if options[:ignore] && options[:include] + raise ArgumentError, ':table_name is required' if options[:include] && !options[:table_name] + + events = options[:events] && ([*options[:events]] & [:insert, :update, :delete]) + events = [:insert, :update, :delete] if events.nil? + raise ArgumentError, ':events should include :insert, :update or :delete' if events && events.empty? + + if options[:include].present? + connection = options.fetch(:connection, ActiveRecord::Base.connection) + + all_table_columns = connection.columns(options[:table_name]).map(&:name) + included_table_columns = options[:include].split(',').map(&:strip).uniq.compact + ignore_list = (all_table_columns - included_table_columns).join(',') + elsif options[:ignore].present? + ignore_list = options[:ignore] + end + + conditions = nil + conditions = "WHEN (#{options[:conditions]})" if options.key?(:conditions) + + params = '' + params << ", 'backup'" if options[:backup] + params << ", #{quote("ignore=#{ignore_list}")}" if ignore_list.present? + params << ", #{quote("pkey=#{Array.wrap(options[:pkey]).join(',')}")}" if options[:pkey].present? + + of_columns = "OF #{options[:of_columns].join(',')}" if options[:of_columns] + + <<-SQL + CREATE TRIGGER %{trigger_name} + AFTER #{events.join(' OR ')} + #{of_columns} + ON %{table_name} + FOR EACH ROW + #{conditions} + EXECUTE PROCEDURE pgq.logutriga(%{queue_name}#{params}); + SQL + end + + def generate_trigger(options = {}) + options = { + table_name: table_name, + connection: work_connection + }.merge(options) + + self.class.generate_trigger(options) + end + + def create_pgq_queue + work_connection.transaction do + result = self.class.pgq_create_queue(pgq_queue_name, work_connection) + raise "Queue already exists! #{pgq_queue_name}" if result == 0 + recreate_trigger(false) + end + end + + def recreate_trigger(lock_table = true) + return unless table_name.present? + + work_connection.transaction do + self.lock_table! if lock_table + drop_pgq_trigger + create_pgq_trigger + end + end + + def pgq_queue_exists? + self.class.pgq_get_queue_info(pgq_queue_name, work_connection).present? + end + + # Public: Рабочее соединение с БД для данной очереди. + # + # В рамках этого соединения производятся все действия с объектами привязанными к данной очереди, + # а именно инициализация и обработка событий. + # + # db_link_class - Имя класса - модели ActiveRecord, обеспечивающая связь с БД. + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def work_connection + return main_connection if db_link_class.nil? + + db_link_class.constantize.connection + end + + # Public: Основное соединение с БД. + # + # В рамках этого соединения производятся общие действия (изменения метаданных). + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def main_connection + ActiveRecord::Base.connection + end + + def pgq_queue_name + "q_#{name}" + end + + protected + + def lock_table! + work_connection.execute <<-SQL + LOCK TABLE #{table_name} IN SHARE MODE + SQL + end + + def create_pgq_trigger(options = {}) + default_options = { + trigger_code: trigger_code || generate_trigger(options), + trigger_name: trigger_name, + table_name: table_name, + queue_name: pgq_queue_name + } + + options.reverse_merge!(default_options) + options[:queue_name] = quote(options[:queue_name]) + + work_connection.execute options[:trigger_code] % options + end + + def drop_pgq_trigger + return unless table_name.present? + + work_connection.execute <<-SQL + DROP TRIGGER IF EXISTS #{trigger_name} ON #{table_name} + SQL + end + + def pgq_drop_queue + return unless self.class.pgq_queue_exists?(pgq_queue_name, work_connection) + self.class.pgq_drop_queue(pgq_queue_name, work_connection) + end + + def trigger_name + # cut scheme name + clear_name = name.split('.').last + "#{TRIGGER_PREFIX}_#{clear_name}" + end + + def quote(text) + self.class.quote(text) + end + + def self.quote(text) + ActiveRecord::Base.connection.quote(text) + end + + def destroy_pgq_queue + work_connection.transaction do + processors.each(&:unregister_consumer) + pgq_drop_queue + drop_pgq_trigger + end + end + end + end +end diff --git a/app/models/treasury/models/supervisor_status.rb b/app/models/treasury/models/supervisor_status.rb new file mode 100644 index 0000000..28759c3 --- /dev/null +++ b/app/models/treasury/models/supervisor_status.rb @@ -0,0 +1,17 @@ +# coding: utf-8 +module Treasury + module Models + class SupervisorStatus < ActiveRecord::Base + self.table_name = 'denormalization.supervisor_status' + self.primary_key = 'id' + + def terminate + update_attribute(:need_terminate, true) + end + + def reset_need_terminate + update_attribute(:need_terminate, false) + end + end + end +end diff --git a/app/models/treasury/models/worker.rb b/app/models/treasury/models/worker.rb new file mode 100644 index 0000000..a37c15f --- /dev/null +++ b/app/models/treasury/models/worker.rb @@ -0,0 +1,24 @@ +# coding: utf-8 +module Treasury + module Models + class Worker < ActiveRecord::Base + self.table_name = 'denormalization.workers' + self.primary_key = 'id' + + has_many :fields, class_name: 'Treasury::Models::Field', inverse_of: :worker + + #validates :name, :presence => true, :length => {:maximum => 25} + #validates_uniqueness_of :name + + scope :active, -> { where(:active => true) } + + def terminate + update_attribute(:need_terminate, true) + end + + def reset_terminate + update_attribute(:need_terminate, false) + end + end + end +end diff --git a/db/migrate/20120313104136_create_schema.rb b/db/migrate/20120313104136_create_schema.rb new file mode 100644 index 0000000..2eae3c8 --- /dev/null +++ b/db/migrate/20120313104136_create_schema.rb @@ -0,0 +1,12 @@ +# coding: utf-8 +class CreateSchema < ActiveRecord::Migration + def up + ActiveRecord::Base.connection.execute "DROP SCHEMA IF EXISTS DENORMALIZATION CASCADE;" + + system %(#{PgTools.psql} < #{File.expand_path(File.join(File.dirname(__FILE__), 'schema.sql'))}) + end + + def down + ActiveRecord::Base.connection.execute "DROP SCHEMA IF EXISTS DENORMALIZATION CASCADE;" + end +end diff --git a/db/migrate/schema.sql b/db/migrate/schema.sql new file mode 100644 index 0000000..72cb3a7 --- /dev/null +++ b/db/migrate/schema.sql @@ -0,0 +1,614 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; + +-- +-- Name: denormalization; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA denormalization; + + +SET search_path = denormalization, pg_catalog; + +SET default_with_oids = false; + +-- +-- Name: events_log; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE events_log ( + id integer NOT NULL, + processed_at timestamp without time zone NOT NULL, + consumer character varying(256) NOT NULL, + event_id character varying(255) NOT NULL, + event_type character varying(1) NOT NULL, + event_time character varying(128) NOT NULL, + event_txid character varying(128) NOT NULL, + event_ev_data text NOT NULL, + event_extra1 text NOT NULL, + event_data text NOT NULL, + event_prev_data text NOT NULL, + event_data_changed boolean NOT NULL, + message character varying(512), + payload character varying(512), + suspected boolean NOT NULL +); + + +-- +-- Name: TABLE events_log; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON TABLE events_log IS 'Отладочный журнал событий'; + + +-- +-- Name: events_log_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE events_log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: events_log_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE events_log_id_seq OWNED BY events_log.id; + + +-- +-- Name: fields; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE fields ( + id integer NOT NULL, + title character varying(128) NOT NULL, + "group" character varying(128) NOT NULL, + field_class character varying(128) NOT NULL, + active boolean DEFAULT false NOT NULL, + need_terminate boolean DEFAULT false NOT NULL, + state character varying(128) DEFAULT 'need_initialize'::character varying NOT NULL, + progress character varying(128), + snapshot_id character varying(4000), + last_error character varying(4000), + worker_id integer, + oid integer NOT NULL, + params character varying(4000), + storage character varying(4000), + pid integer +); + + +-- +-- Name: TABLE fields; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON TABLE fields IS 'Поля'; + + +-- +-- Name: COLUMN fields.title; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.title IS 'Название поля'; + + +-- +-- Name: COLUMN fields."group"; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields."group" IS 'Группа поля'; + + +-- +-- Name: COLUMN fields.field_class; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.field_class IS 'Класс поля'; + + +-- +-- Name: COLUMN fields.active; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.active IS 'Активно/не активно'; + + +-- +-- Name: COLUMN fields.need_terminate; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.need_terminate IS 'Флаг необходимости прекращения текущей операции'; + + +-- +-- Name: COLUMN fields.state; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.state IS 'Состояние поля'; + + +-- +-- Name: COLUMN fields.progress; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.progress IS 'Прогресс иницилизации'; + + +-- +-- Name: COLUMN fields.snapshot_id; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.snapshot_id IS 'ИД снапшота БД, в котором проводилась иницилизация'; + + +-- +-- Name: COLUMN fields.last_error; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.last_error IS 'Последняя ошибка'; + + +-- +-- Name: COLUMN fields.worker_id; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.worker_id IS 'Ссылка на воркера (worker.id)'; + + +-- +-- Name: COLUMN fields.oid; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN fields.oid IS 'Порядковый номер поля'; + + +-- +-- Name: fields_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE fields_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fields_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE fields_id_seq OWNED BY fields.id; + + +-- +-- Name: processors; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE processors ( + id integer NOT NULL, + queue_id integer NOT NULL, + field_id integer NOT NULL, + processor_class character varying(128) NOT NULL, + oid integer NOT NULL, + params character varying(2000), + consumer_name character varying(128) NOT NULL +); + + +-- +-- Name: TABLE processors; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON TABLE processors IS 'Процессоры - обработчики'; + + +-- +-- Name: COLUMN processors.queue_id; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN processors.queue_id IS 'Ссылка на очередь (queues.id)'; + + +-- +-- Name: COLUMN processors.field_id; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN processors.field_id IS 'Ссылка на поле (fields.id)'; + + +-- +-- Name: COLUMN processors.processor_class; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN processors.processor_class IS 'Класс обработчика'; + + +-- +-- Name: COLUMN processors.oid; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN processors.oid IS 'Порядковый номер обработчика в рамках поля'; + + +-- +-- Name: COLUMN processors.params; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN processors.params IS 'Параметры обработчика'; + + +-- +-- Name: processors_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE processors_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: processors_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE processors_id_seq OWNED BY processors.id; + + +-- +-- Name: queues; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE queues ( + id integer NOT NULL, + name character varying(128) NOT NULL, + table_name character varying(256), + trigger_code character varying(2000), + db_link_class character varying(256) +); + + +-- +-- Name: TABLE queues; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON TABLE queues IS 'Очереди'; + + +-- +-- Name: COLUMN queues.name; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN queues.name IS 'Название'; + + +-- +-- Name: COLUMN queues.table_name; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN queues.table_name IS 'Имя таблицы'; + + +-- +-- Name: COLUMN queues.trigger_code; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN queues.trigger_code IS 'Код триггера'; + + +-- +-- Name: COLUMN queues.db_link_class; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN queues.db_link_class IS 'Имя класс - модели ActiveRecord, обеспечивающего соединение с БД, в рамках которого обрабатывается очередь (опционально)'; + + +-- +-- Name: queues_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE queues_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: queues_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE queues_id_seq OWNED BY queues.id; + + +-- +-- Name: supervisor_status; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE supervisor_status ( + id integer NOT NULL, + active boolean DEFAULT false NOT NULL, + state character varying(128) DEFAULT 'stopped'::character varying NOT NULL, + need_terminate boolean DEFAULT false NOT NULL, + last_error character varying(4000), + pid integer +); + + +-- +-- Name: COLUMN supervisor_status.active; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN supervisor_status.active IS 'Активно/не активно'; + + +-- +-- Name: COLUMN supervisor_status.state; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN supervisor_status.state IS 'Состояние поля'; + + +-- +-- Name: COLUMN supervisor_status.need_terminate; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN supervisor_status.need_terminate IS 'Флаг необходимости прекращения текущей операции'; + + +-- +-- Name: supervisor_status_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE supervisor_status_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: supervisor_status_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE supervisor_status_id_seq OWNED BY supervisor_status.id; + + +-- +-- Name: workers; Type: TABLE; Schema: denormalization; Owner: - +-- + +CREATE TABLE workers ( + id integer NOT NULL, + active boolean DEFAULT false NOT NULL, + state character varying(128) DEFAULT 'stopped'::character varying NOT NULL, + need_terminate boolean DEFAULT false NOT NULL, + last_error character varying(4000), + name character varying(25) NOT NULL, + pid integer +); + + +-- +-- Name: TABLE workers; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON TABLE workers IS 'Воркеры'; + + +-- +-- Name: COLUMN workers.active; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN workers.active IS 'Активно/не активно'; + + +-- +-- Name: COLUMN workers.state; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN workers.state IS 'Состояние поля'; + + +-- +-- Name: COLUMN workers.need_terminate; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN workers.need_terminate IS 'Флаг необходимости прекращения текущей операции'; + + +-- +-- Name: COLUMN workers.last_error; Type: COMMENT; Schema: denormalization; Owner: - +-- + +COMMENT ON COLUMN workers.last_error IS 'Последняя ошибка'; + + +-- +-- Name: workers_id_seq; Type: SEQUENCE; Schema: denormalization; Owner: - +-- + +CREATE SEQUENCE workers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: workers_id_seq; Type: SEQUENCE OWNED BY; Schema: denormalization; Owner: - +-- + +ALTER SEQUENCE workers_id_seq OWNED BY workers.id; + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY events_log ALTER COLUMN id SET DEFAULT nextval('events_log_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY fields ALTER COLUMN id SET DEFAULT nextval('fields_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY processors ALTER COLUMN id SET DEFAULT nextval('processors_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY queues ALTER COLUMN id SET DEFAULT nextval('queues_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY supervisor_status ALTER COLUMN id SET DEFAULT nextval('supervisor_status_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY workers ALTER COLUMN id SET DEFAULT nextval('workers_id_seq'::regclass); + + +-- +-- Name: events_log_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY events_log + ADD CONSTRAINT events_log_pkey PRIMARY KEY (id); + + +-- +-- Name: fields_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY fields + ADD CONSTRAINT fields_pkey PRIMARY KEY (id); + + +-- +-- Name: processors_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY processors + ADD CONSTRAINT processors_pkey PRIMARY KEY (id); + + +-- +-- Name: queues_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY queues + ADD CONSTRAINT queues_pkey PRIMARY KEY (id); + + +-- +-- Name: supervisor_status_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY supervisor_status + ADD CONSTRAINT supervisor_status_pkey PRIMARY KEY (id); + + +-- +-- Name: workers_pkey; Type: CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY workers + ADD CONSTRAINT workers_pkey PRIMARY KEY (id); + + +-- +-- Name: uq_fields_class; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_fields_class ON fields USING btree (field_class); + + +-- +-- Name: uq_fields_consumer_name; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_fields_consumer_name ON processors USING btree (consumer_name); + + +-- +-- Name: uq_fields_title; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_fields_title ON fields USING btree (title); + + +-- +-- Name: uq_processors; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_processors ON processors USING btree (processor_class, params); + + +-- +-- Name: uq_queues; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_queues ON queues USING btree (name, table_name); + + +-- +-- Name: uq_workers_name; Type: INDEX; Schema: denormalization; Owner: - +-- + +CREATE UNIQUE INDEX uq_workers_name ON workers USING btree (name); + + +-- +-- Name: fk_processors_field; Type: FK CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY processors + ADD CONSTRAINT fk_processors_field FOREIGN KEY (field_id) REFERENCES fields(id) DEFERRABLE; + + +-- +-- Name: fk_processors_queue; Type: FK CONSTRAINT; Schema: denormalization; Owner: - +-- + +ALTER TABLE ONLY processors + ADD CONSTRAINT fk_processors_queue FOREIGN KEY (queue_id) REFERENCES queues(id) DEFERRABLE; + + +-- +-- PostgreSQL database dump complete +-- diff --git a/docker-compose.yml b/docker-compose.yml index 99f90a0..ab05ad7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,11 +6,21 @@ services: environment: - BUNDLE_PATH=/bundle/$DOCKER_RUBY_VERSION - SSH_AUTH_SOCK=/ssh/auth/sock + - TEST_DB_HOST=db + - TEST_DB_NAME=docker + - TEST_DB_USERNAME=postgres command: bash + depends_on: + - db volumes: - .:/app - ssh_data:/ssh:ro + db: + image: abakpress/postgres:$POSTGRES_IMAGE_TAG + environment: + - POSTGRES_DB=docker + volumes: ssh_data: external: true diff --git a/lib/tasks/denormalization.rake b/lib/tasks/denormalization.rake new file mode 100644 index 0000000..3c66534 --- /dev/null +++ b/lib/tasks/denormalization.rake @@ -0,0 +1,31 @@ +# coding: utf-8 +namespace :denormalization do + desc "restart denormalization" + task restart: :environment do + Treasury::Controller.restart + end + + desc "stop supervisor and all workers" + task stop: :environment do + Treasury::Controller.stop + end + + task start: :restart + + desc "stop supervisor" + namespace :supervisor do + task stop: :environment do + Treasury::Controller.stop_supervisor + end + end + + desc "Пересоздает триггеры для всех очередей" + task recreate_triggers: :environment do + Treasury::Models::Queue.all.each(&:recreate_trigger) + end + + desc "Удаляет все триггеры системы денормализации" + task drop_triggers: :environment do + Apress::Utils::Triggers.drop_triggers(:trigger_pattern => "#{Treasury::Models::Queue::TRIGGER_PREFIX}*") + end +end diff --git a/lib/tasks/events_logger.rake b/lib/tasks/events_logger.rake new file mode 100644 index 0000000..b90bfe6 --- /dev/null +++ b/lib/tasks/events_logger.rake @@ -0,0 +1,41 @@ +# coding: utf-8 +namespace :denormalization do + namespace :events_logger do + desc 'Перенос накопленных, за сутки, данных из Redis в БД' + task daily: :environment do + date = ENV['date'] || (Date.today - 1.day) + if date.is_a?(String) + date_arr = date.split('-') + date = Date.civil(date_arr[0].to_i, date_arr[1].to_i, date_arr[2].to_i) + end + + trem = ENV['trem'].nil? ? false : ENV['trem'] == 'true' + rrem = ENV['rrem'].nil? ? true : ENV['rrem'] == 'true' + + Treasury::Services::EventsLogger.new.process(date, trem, rrem) + end + + desc 'Удаление накопленных, за сутки, данных из Redis' + task delete: :environment do + date = ENV['date'] || (Date.today - 1.day) + if date.is_a?(String) + date_arr = date.split('-') + date = Date.civil(date_arr[0].to_i, date_arr[1].to_i, date_arr[2].to_i) + end + + Treasury::Services::EventsLogger.new.delete_events(date) + end + + desc 'Показать список дат, за которые есть данные в Redis' + task dates: :environment do + dates = Treasury::Services::EventsLogger.new.dates_list + if !dates.empty? + dates.each do |date, count| + puts "#{date} - #{count} log rows" + end + else + puts 'Empty.' + end + end + end +end diff --git a/lib/tasks/pgq.rake b/lib/tasks/pgq.rake new file mode 100644 index 0000000..cb54242 --- /dev/null +++ b/lib/tasks/pgq.rake @@ -0,0 +1,11 @@ +# coding: utf-8 + +namespace :pgq do + desc "Перезалить схему pgq" + task recreate_schema: :environment do + raise "Don't run this task in production environment!" if Rails.env.production? + + ActiveRecord::Base.connection.execute "DROP SCHEMA IF EXISTS pgq CASCADE;" + system %(#{PgTools.psql} < #{Rails.root}/db/pgq/schema.sql) + end +end diff --git a/lib/tasks/treasury.rake b/lib/tasks/treasury.rake new file mode 100644 index 0000000..aa8f50b --- /dev/null +++ b/lib/tasks/treasury.rake @@ -0,0 +1,14 @@ +namespace :treasury do + task stop: :environment do + Treasury::Controller.stop + Treasury::BgExecutor.daemonize("stop") + end + + task restart: :environment do + Treasury::BgExecutor.daemonize("restart") + ActiveRecord::Base.connection.reconnect! + Treasury::Controller.restart + end + + task start: :restart +end diff --git a/lib/treasury.rb b/lib/treasury.rb index 26a8b39..eda0154 100644 --- a/lib/treasury.rb +++ b/lib/treasury.rb @@ -6,8 +6,14 @@ require 'treasury/engine' require 'treasury/bg_executor' +require 'resque-integration' +require 'string_tools' +require 'pg_tools' + module Treasury LIST_DELIMITER = ','.freeze + ROOT_REDIS_KEY = 'denormalization'.freeze + ROOT_LOGGER_DIR = 'denormalization'.freeze def self.configuration @configuration ||= Configuration.new diff --git a/lib/treasury/backwards.rb b/lib/treasury/backwards.rb new file mode 100644 index 0000000..224eba2 --- /dev/null +++ b/lib/treasury/backwards.rb @@ -0,0 +1,121 @@ +module CoreDenormalization + module Fields + module Company + Base = ::Treasury::Fields::Company::Base + end + + module User + Base = ::Treasury::Fields::User::Base + end + + module Product + Base = ::Treasury::Fields::Product::Base + end + + Base = ::Treasury::Fields::Base + Single = ::Treasury::Fields::Single + Translator = ::Treasury::Fields::Translator + Delayed = ::Treasury::Fields::Delayed + Errors = ::Treasury::Fields::Errors + NoRequireInitialization = ::Treasury::Fields::NoRequireInitialization + end + + module Processors + module Company + Base = ::Treasury::Processors::Company::Base + end + + module User + Base = ::Treasury::Processors::User::Base + end + + Base = ::Treasury::Processors::Base + Single = ::Treasury::Processors::Single + Translator = ::Treasury::Processors::Translator + OptimizedTranslator = ::Treasury::Processors::OptimizedTranslator + Product = ::Treasury::Processors::Product + Delayed = ::Treasury::Processors::Delayed + Errors = ::Treasury::Processors::Errors + Counter = ::Treasury::Processors::Counter + Counters = ::Treasury::Processors::Counters + end + + module Storage + Base = ::Treasury::Storage::Base + + module PostgreSQL + Base = ::Treasury::Storage::PostgreSQL::Base + Db = ::Treasury::Storage::PostgreSQL::Db + PgqProducer = ::Treasury::Storage::PostgreSQL::PgqProducer + end + + module Redis + Base = ::Treasury::Storage::Redis::Base + Multi = ::Treasury::Storage::Redis::Multi + end + end + + ROOT_REDIS_KEY = ::Treasury::ROOT_REDIS_KEY +end + +module Denormalization + module Fields + module Company + Base = ::Treasury::Fields::Company::Base + end + + module User + Base = ::Treasury::Fields::User::Base + end + + module Product + Base = ::Treasury::Fields::Product::Base + end + + Base = ::Treasury::Fields::Base + Single = ::Treasury::Fields::Single + Translator = ::Treasury::Fields::Translator + Delayed = ::Treasury::Fields::Delayed + Errors = ::Treasury::Fields::Errors + NoRequireInitialization = ::Treasury::Fields::NoRequireInitialization + end + + module Processors + module Company + Base = ::Treasury::Processors::Company::Base + end + + module User + Base = ::Treasury::Processors::User::Base + end + + Base = ::Treasury::Processors::Base + Single = ::Treasury::Processors::Single + Translator = ::Treasury::Processors::Translator + OptimizedTranslator = ::Treasury::Processors::OptimizedTranslator + Product = ::Treasury::Processors::Product + Delayed = ::Treasury::Processors::Delayed + Errors = ::Treasury::Processors::Errors + Counter = ::Treasury::Processors::Counter + Counters = ::Treasury::Processors::Counters + end + + module Storage + Base = ::Treasury::Storage::Base + + module PostgreSQL + Base = ::Treasury::Storage::PostgreSQL::Base + Db = ::Treasury::Storage::PostgreSQL::Db + PgqProducer = ::Treasury::Storage::PostgreSQL::PgqProducer + end + + module Redis + Base = ::Treasury::Storage::Redis::Base + Multi = ::Treasury::Storage::Redis::Multi + end + end + + ROOT_REDIS_KEY = ::Treasury::ROOT_REDIS_KEY +end + +Pgq = ::Treasury::Pgq diff --git a/lib/treasury/bg_executor/job.rb b/lib/treasury/bg_executor/job.rb index ef24cbb..4c015ea 100644 --- a/lib/treasury/bg_executor/job.rb +++ b/lib/treasury/bg_executor/job.rb @@ -53,15 +53,6 @@ def acts_as_critical? !!critical_job end - # указать, что не нужно вести трассировку сессии для NewRelic - def acts_as_no_trace - self.no_trace_job = true - end - - def acts_as_no_trace? - !!no_trace_job - end - # указать, что не нужно вести трассировку сессии для NewRelic def acts_as_no_cancel self.no_cancel_job = true diff --git a/lib/treasury/configuration.rb b/lib/treasury/configuration.rb index d671a73..4953116 100644 --- a/lib/treasury/configuration.rb +++ b/lib/treasury/configuration.rb @@ -5,7 +5,8 @@ class Configuration :bge_max_tries_on_fail, :bge_namespace, :bge_queue_timeout, - :bge_daemon_options + :bge_daemon_options, + :job_error_notifications def initialize self.bge_concurrency = 4 diff --git a/lib/treasury/controller.rb b/lib/treasury/controller.rb new file mode 100644 index 0000000..359a02e --- /dev/null +++ b/lib/treasury/controller.rb @@ -0,0 +1,153 @@ +# coding: utf-8 + +module Treasury + class Controller + SUPERVISOR_TERMINATE_TIMEOUT = 10 # seconds + SUPERVISOR_JOB_NAME = 'treasury/supervisor'.freeze + SUPERVISOR_CMDLINE_PATTERN = ': treasury/supervisor'.freeze + WORKERS_TERMINATE_TIMEOUT = 60 # seconds + WORKER_CMDLINE_PATTERN = ': treasury/worker'.freeze + WORKER_JOB_NAME = 'treasury/worker'.freeze + + class << self + # Start Supervisor + def start + puts 'Starting denormalization service...' + + unless supervisor + puts 'No Supervisor configured.' + return + end + + if bg_executor_client.singleton_job_running?(SUPERVISOR_JOB_NAME, []) + puts 'Supervisor is already running' + else + puts 'run...' + job_id, job_key = bg_executor_client.queue_job!(SUPERVISOR_JOB_NAME) + puts 'Supervisor successfully running, job_id = %s, job_key = %s' % [job_id, job_key] + end + end + + # Stop Supervisor and all Workers + def stop + puts 'Stopping denormalization service...' + + unless supervisor + puts 'No Supervisor configured.' + return + end + + stop_supervisor + + terminate_all_workers + reset_all_workers_jobs + + true + end + + # Restart Supervisor and all Workers + def restart + puts 'Restarting denormalization service...' + + unless supervisor + puts 'No Supervisor configured.' + return + end + + stop + start + end + + def stop_supervisor + unless supervisor + puts 'No Supervisor configured.' + return + end + + if supervisor_pid.present? + puts 'Supervisor is running. Send command to stop...' + supervisor.terminate + + begin + Timeout.timeout(SUPERVISOR_TERMINATE_TIMEOUT) do + sleep(5.seconds) while supervisor_pid.present? + puts 'Supervisor stopped.' + end + rescue Timeout::Error + puts 'Timeout expired. Terminating...' + terminate_supervisor + end + else + puts 'Supervisor is not running.' + end + + puts 'Reset supervisor job state...' + supervisor.reset_need_terminate + reset_supervisor_job + end + + # Дает команду на завершение работы всем рабочим процессам + def terminate_all_workers + puts 'Terminate all workers...' + + begin + Timeout.timeout(WORKERS_TERMINATE_TIMEOUT) do + while workers_pids.present? + Treasury::Models::Worker.all.map(&:terminate) + sleep(5.seconds) + end + puts 'Workers stopped.' + end + rescue Timeout::Error + puts 'Timeout expired. Terminating...' + `kill -9 #{workers_pids}` + end + end + + private + + def workers_pids + `pgrep -f "#{WORKER_CMDLINE_PATTERN}" &2>/dev/null`.gsub("\n", ' ') + end + + def supervisor_pid + `pgrep -f "#{SUPERVISOR_CMDLINE_PATTERN}" &2>/dev/null` + end + + def reset_supervisor_job + bg_executor_client.send(:remove_from_singletons, + bg_executor_client.job_class(SUPERVISOR_JOB_NAME).singleton_hexdigest({})) + end + + def reset_all_workers_jobs + puts 'Reset workers jobs state...' + Treasury::Models::Worker.all.each do |worker| + bg_executor_client.send( + :remove_from_singletons, + bg_executor_client.job_class(WORKER_JOB_NAME).singleton_hexdigest(worker_id: worker.id) + ) + end + end + + def bg_executor_client + Treasury::BgExecutor::Client.instance + end + + def terminate_supervisor + pid = supervisor_pid + return unless pid.present? + + `kill -9 #{pid}` + + Timeout.timeout(SUPERVISOR_TERMINATE_TIMEOUT) do + sleep(5.seconds) while supervisor_pid.present? + puts 'Supervisor terminated.' + end + end + + def supervisor + Treasury::Models::SupervisorStatus.first + end + end + end +end diff --git a/lib/treasury/engine.rb b/lib/treasury/engine.rb index 1a50671..7c198e2 100644 --- a/lib/treasury/engine.rb +++ b/lib/treasury/engine.rb @@ -5,5 +5,18 @@ module Treasury class Engine < ::Rails::Engine config.autoload_paths += Dir["#{config.root}/lib"] + + initializer 'treasury', before: :load_init_rb do |app| + app.config.paths['db/migrate'].concat(config.paths['db/migrate'].expanded) + + ActiveRecord::Base.extend(Treasury::Pgq) if defined?(ActiveRecord) + require 'treasury/backwards' + end + + initializer 'treasury-factories', after: 'factory_girl.set_factory_paths' do |_| + if defined?(FactoryGirl) + FactoryGirl.definition_file_paths.unshift root.join('spec', 'factories') + end + end end end diff --git a/lib/treasury/fields.rb b/lib/treasury/fields.rb new file mode 100644 index 0000000..b6a5651 --- /dev/null +++ b/lib/treasury/fields.rb @@ -0,0 +1,7 @@ +module Treasury + module Fields + STATE_NEED_INITIALIZE = 'need_initialize'.freeze + STATE_IN_INITIALIZE = 'in_initialize'.freeze + STATE_INITIALIZED = 'initialized'.freeze + end +end diff --git a/lib/treasury/fields/accessors.rb b/lib/treasury/fields/accessors.rb new file mode 100644 index 0000000..5c19cd7 --- /dev/null +++ b/lib/treasury/fields/accessors.rb @@ -0,0 +1,33 @@ +# coding: utf-8 + +module Treasury + module Fields + module Accessors + def value_as_string(params) + raise_no_implemented(:string, params) + end + + def value_as_integer(params) + raise_no_implemented(:integer, params) + end + + def value_as_boolean(params) + raise_no_implemented(:boolean, params) + end + + def value_as_date(params) + raise_no_implemented(:date, params) + end + + def value_as_array(params) + raise_no_implemented(:array, params) + end + + private + + def raise_no_implemented(accessor_type, params) + raise Errors::NoAccessor.new(self, accessor_type, params) + end + end + end +end diff --git a/lib/treasury/fields/base.rb b/lib/treasury/fields/base.rb index 46db0a6..fc44c28 100644 --- a/lib/treasury/fields/base.rb +++ b/lib/treasury/fields/base.rb @@ -1,6 +1,543 @@ +# coding: utf-8 + module Treasury module Fields - class Base < ::CoreDenormalization::Fields::Base + class Base + include Treasury::Utils + include Treasury::Session + include Treasury::Logging + include ActiveSupport::Callbacks + extend Accessors + + DEFAULT_BATCH_SIZE = 1000 + + # пауза после обработки батча данных + # позволяет снизить нагрузку на БД при инициализации, + # за счет увеличения времени инициализации + DEFAULT_BATCH_PAUSE = Rails.env.staging? || !Rails.env.production? ? 0 : 2.seconds + + STATE_CACHE_TTL = 1.hour + + LOGGER_FILE_NAME = "#{ROOT_LOGGER_DIR}/field_initializer".freeze + + class_attribute :_instance + + class_attribute :initialize_method + + self.initialize_method = :offset + + attr_accessor :batch_size + attr_reader :snapshot_id + + # Коллбек изменения данных поля. + # Срабатывает после полной записи во все хранилища и после подтверждения транзакции в них. + # Атрибут экземпляра changed_objects, содержит массив идентификаторов измененных объектов + # + # Example: + # module FieldCallback + # extend ActiveSupport::Concern + # + # included do + # set_callback :data_changed, :after do |field| + # Apress::Companies::Sweeper.expire(field.changed_objects) + # end + # end + # end + # + # FieldClass.send(:include, FieldCallback) + # + define_callbacks :data_changed + + def self.create_by_class(klass, field_model = Treasury::Models::Field.find_by_field_class(klass.to_s)) + raise Errors::UnknownFieldClassError if field_model.nil? + return klass.to_s.constantize.new(field_model) + rescue => e + log_error(e) + raise + end + + def self.instance + # TODO: нужно будет придумать механизм инвалидации инстанса при изменении параметров хранилища и т.п. + self._instance ||= create_by_class(self) + end + + def initialize(field_model) + @process_object = field_model + init_params + end + + def init_params + @batch_size = DEFAULT_BATCH_SIZE + @batch_pause = DEFAULT_BATCH_PAUSE + end + + def initialize! + logger.info "Процесс иницилизации поля #{quote(field_model.title)} запущен" + return unless check_active + return unless check_terminate + return unless check_need_initialize + return unless check_processors + + set_state(STATE_IN_INITIALIZE) + clear_last_error + subscribe_all_consumer + reset_storage_data + initialize_field + rescue => e + log_error(e) + set_state(STATE_NEED_INITIALIZE) rescue nil + raise + ensure + logger.info "Процесс иницилизации поля остановлен" + end + + def storages + @storages ||= storages_hash.values + end + + # Public: Возвращает Storage-объекты с идентификатором из переданного множества + # + # ids - Array of Symbols, массив идентификаторов хранилищ + # + # Returns Array of Storage + def storages_by_ids(ids) + storages.select { |storage| ids.include?(storage.id) } + end + + def default_storage + storages.first + end + + def write_data(data, force = false) + return if data.empty? + storages.each do |storage| + storage.transaction_bulk_write(data) if force || !storage.params[:write_only_processed] + end + end + + def field_model + @process_object + end + + def first_field + # params[:fields] - хэш полей и их параметров + # каждый элемент хэш: field => {params} + @first_field ||= field_params[:fields].keys.first.to_sym + end + + # Public: Возвращает значение поля. + # + # object - String или Integer идентификатор объекта, для которого запрашивается значение. + # field - String или Symbol идентификатор поля, значение которого запрашивается. + # Не обязательный параметр. По умолчанию - первое поле. + # storage - String или Symbol идентификатор хранилища, из которого нужно получить значение + # Не обязательный параметр. По умолчанию - хранилище по умолчанию. + # + # Examples: + # Apress::AwesomeField.raw_value(user.id, :products_count) + # # => 10 + # + # Returns значение поля. Тип зависит от реализации конкретного хранилища. + # Raises UninitializedFieldError, если поле не инициализировано. + + def raw_value(object, field = nil, storage = nil) + raise Errors::UninitializedFieldError unless cached_state == STATE_INITIALIZED + raw_value_from_storage object, field, storage + end + + # Public: Проверяет инициализировано поле или нет. + # Если нет, возвращает nil. Если да, возвращает значение поля. + # Returns + # Значение поля - если инициализировано + # nil - если не инициализировано + def raw_value?(object, field = nil, storage = nil) + return unless cached_state == STATE_INITIALIZED + raw_value_from_storage object, field, storage + end + + protected + + attr_reader :changed_objects + + def self.value + raw_value(@accessing_object, @accessing_field) + end + + def self.value? + raw_value?(@accessing_object, @accessing_field) + end + + def raw_value_from_storage(object, field = nil, storage = nil) + storage = storages_hash[storage || default_storage.id] + field ||= first_field + storage.read(object, field) + end + + def fields_for_reset + field_params[:fields].keys + end + + def reset_storage_data + storages.each do |storage| + logger.info "Сбрасываю данные хранилища #{storage.id}" + storage.reset_data(objects_for_reset, fields_for_reset) + end + end + + # Protected: Возвращает список идентификаторов объектов для сброса. + # + # Если доступен метод reset_statement, то выполняет запрос и возвращает список объектов, + # иначе возвращает nil. + # + # Returns Array or nil. + # + def objects_for_reset + return nil unless respond_to?(:reset_statement) + + logger.info "Запрашиваю список объектов для сброса:\r\n #{reset_statement}" + work_connection.select_values(reset_statement) + end + + def initialize_field + logger.info "Стартую spanshot-транзакцию" + + main_connection.transaction do + work_connection.transaction do + lock_storages + + @snapshot_id = start_snapshot + + before_initialize + + @total_rows = 0 + + case self.class.initialize_method + when :offset then offset_initialize + when :interval then interval_initialize + else raise 'Unknown initialize method' + end + + save_snapshot + set_state(STATE_INITIALIZED) + + after_initialize + end + end + end + + # Офсетная иниализация + def offset_initialize + step = 0 + while true + exit unless check_terminate + + step += 1 + offset = (step - 1) * batch_size + + count_rows = fetch_rows(:query_args => {:offset => offset, :limit => batch_size}) + + break if count_rows < batch_size + end + end + + # Интервальная инициализация + def interval_initialize + min_id, max_id = work_connection.select_one(interval_statement).values.map(&:to_i) + return if min_id.nil? || max_id.nil? + + logger.info "min_id = #{min_id}, max_id = #{max_id}, ~ count = #{max_id - min_id}" + + while min_id < max_id + exit unless check_terminate + + next_id = min_id + batch_size - 1 + fetch_rows(:query_args => {:min_id => min_id, :max_id => next_id}) + min_id = next_id + 1 + end + end + + # Затянуть строчки для обработки + # + # options - Hash + # :query_args - Hash + # + # Return Integer size of fetched rows + def fetch_rows(options) + before_batch_process + rows = query_rows(options[:query_args]) + after_batch_process + + @total_rows += rows.size + + write_data(process_rows(rows)) unless rows.count.zero? + save_progress(@total_rows) + + sleep(@batch_pause) + + rows.size + end + + # Запрос на получение срочек + # + # query_args - Hash + # + # Returns Array + def query_rows(query_args) + query = initialize_statement % query_args + logger.info "Выполняю запрос:\r\n %s" % query + work_connection.select_all(query).map do |row| + HashWithIndifferentAccess.new(row) + end + end + + def process_rows(rows) + rows.inject(HashWithIndifferentAccess.new) do |hash, row| + hash.merge!(row[:object_id] => row.except(:object_id)) + end + end + + def initialize_statement + raise NotImplementedError + end + + def start_snapshot + work_connection.select_value <<-SQL + SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT txid_current_snapshot(); + SQL + end + + def lock_storages + logger.info "Лочу хранилища" + storages.each(&:lock) + end + + def before_initialize + return unless defined?(prepare_data_block) + logger.info "Выполняю блок подготовки данных:\r\n %s" % prepare_data_block + work_connection.execute(prepare_data_block) + end + + # Protected: Выполняет блок финализации данных, если он задан. + # + # * Выполняется только в случае успешной инициализации. + # + # Returns nothing. + + def after_initialize + return unless defined?(finalize_data_block) + logger.info "Выполняю блок финализации данных:\r\n %s" % finalize_data_block + work_connection.execute(finalize_data_block) + end + + def before_batch_process + return unless defined?(prepare_batch_data_block) + logger.info "Выполняю блок подготовки пачки:\r\n %s" % prepare_batch_data_block + work_connection.execute(prepare_batch_data_block) + end + + # Protected: Выполняет блок финализации данных пачки, если он задан. + # + # * Выполняется только в случае успешной обработки порции данных. + # + # Returns nothing. + + def after_batch_process + return unless defined?(finalize_batch_data_block) + logger.info "Выполняю блок финализации пачки:\r\n %s" % finalize_batch_data_block + work_connection.execute(finalize_batch_data_block) + end + + def lock_table(processor) + table_name = processor.queue.table_name + return if table_name.blank? + logger.info "Лочу таблицу #{table_name}" + work_connection.execute <<-SQL + LOCK TABLE #{table_name} IN SHARE MODE + SQL + end + + def subscribe_all_consumer + work_connection.transaction do + field_model.processors.each do |processor| + logger.info "Иницилизация процессора %s" % processor.processor_class + lock_table(processor) + logger.info "Подписываюсь на события" + processor.subscribe! + end + end + end + + def save_snapshot + field_model.snapshot_id = @snapshot_id + field_model.save! + end + + def save_progress(rows_processed) + logger.info "Обработано строк: #{rows_processed}" + end + + def check_active + return true if field_model.active? + logger.warn "Поле не активно" + false + end + + def check_need_initialize + return true if check_state(STATE_NEED_INITIALIZE) + return true if check_state(STATE_IN_INITIALIZE) && process_is_dead?(field_model.pid) + logger.warn "Поле не требует иницилизации" + false + end + + def check_processors + return true if field_model.processors.exists? + logger.warn "Поле не имеет не одного процессора" + false + end + + # Public: Возвращает значение поля. + # + # See InstanceMethods#raw_value + # + # Returns значение поля. + + def self.raw_value(object, field = nil, storage = nil) + instance.raw_value(object, field, storage) + end + + # Public: Проверяет инициализировано ли поле. + # Если нет, то возвращает nil. + # Если да, то возвращает значение поля. + # + # See InstanceMethods#raw_value? + # + # Returns + # nil - если поле не инициализировано + # значение поля - если поле ициализировано + def self.raw_value?(object, field = nil, storage = nil) + instance.raw_value?(object, field, storage) + end + + def self.logger_default_file_name + LOGGER_FILE_NAME + end + + def quote(str) + ::ActiveRecord::Base.quote_value(str) + end + + def self.extract_object(params) + raise NotImplementedError + end + + def self.init_accessor(params) + @accessing_object = extract_object(params) + @accessing_field = params[:field] + end + + def self.accessing_object + @accessing_object + end + + def self.accessing_field + @accessing_field + end + + def data_changed(changed_objects) + @changed_objects = changed_objects + run_callbacks(:data_changed, :after) + end + + # Public: Рабочее соединение с БД для данной очереди. + # + # В рамках этого соединения производятся все действия с объектами привязанными к данной очереди, + # а именно инициализация и обработка событий. + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def work_connection + @work_connection ||= field_model.processors.first.queue.work_connection + end + + # Public: Основное соединение с БД. + # + # В рамках этого соединения производятся общие действия (изменения метаданных). + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def main_connection + ActiveRecord::Base.connection + end + + private + + attr_accessor :state_updated_at + + # Private: Возвращает хранилища поля. + # + # * У поля может быть несколько хранилищ. + # field.storage - это массив хэшей вида: {:class, :id, :params} + # * Хранилище по умолчанию - первое хранилище + # + # Returns HashWithIndifferentAccess всех хранилищ поля. + # Каждый элемент хэша - это экземпляр хранилища. + # {storage_id => storage_instance} + + def storages_hash + @storages_hash ||= field_model.storage.inject(HashWithIndifferentAccess.new) do |hash, storage| + params = storage.fetch(:params, Hash.new) + params[:id] = storage[:id] if storage[:id] + params.reverse_merge!(storage_params) + + storage = storage[:class].constantize.new(params) + storage.source = self + hash.merge!(storage.id => storage) + end + end + + def storage_params + params = { + :fields => field_params[:fields], + :fields_prefix => fields_storage_prefix + } + + params[:reset_strategy] = field_params[:reset_strategy] if field_params.key?(:reset_strategy) + params + end + + # Private: Возвращает кэшированное состояние поля. + # + # * Состояние поле кэшируется на время STATE_CACHE_TTL + # + # Returns Fields::STATES*. + + def cached_state + if @state_updated_at.nil? || (Time.now - @state_updated_at >= STATE_CACHE_TTL) + @state_updated_at = Time.now + refresh_state + end + + field_model.state + end + + # Private: Возвращает параметры Поля. + # + # Returns Hash. + + def field_params + field_model.params.with_indifferent_access || {} + end + + # Private: Возвращает префикс хранения для полей. + # + # Returns String. + + def fields_storage_prefix + prefix = field_params[:fields_storage_prefix] + return if prefix.blank? + "#{prefix}_" + end end end end diff --git a/lib/treasury/fields/company/base.rb b/lib/treasury/fields/company/base.rb index 9bbc1eb..78327f4 100644 --- a/lib/treasury/fields/company/base.rb +++ b/lib/treasury/fields/company/base.rb @@ -3,7 +3,56 @@ module Treasury module Fields module Company - class Base < ::CoreDenormalization::Fields::Company::Base + # + # Базовый класс - Поле системы денормализации, для полей на основе компании. + # + class Base < Treasury::Fields::Base + BATCH_SIZE = 10_000 + + protected + + # Protected: Инициализирует параметры поля. + # + # Returns nothing. + + def init_params + super + self.batch_size = BATCH_SIZE + end + + # Protected: Возвращает идентификатор компании, переданой как объект в параметрах. + # + # params - Hash параметров: + # :object - String/Numeric/::Company или Hash, содержащий элемент + # :company или :company_id, указанных типов. + # + # Returns Numeric. + + def self.extract_object(params) + company = params[:object] + company = company[:company] || company[:company_id] if company.is_a?(Hash) + + case company + when ::Numeric + company + when ::String + company.to_i + else + if company && company.respond_to?(:id) + company.id + else + raise ArgumentError, "Company instance or Numeric/String company id expected!', #{params.inspect}" + end + end + end + + class << self + alias :extract_company :extract_object + end + + class << self + alias :accessing_company :accessing_object + end end end end diff --git a/lib/treasury/fields/delayed.rb b/lib/treasury/fields/delayed.rb new file mode 100644 index 0000000..132253d --- /dev/null +++ b/lib/treasury/fields/delayed.rb @@ -0,0 +1,18 @@ +# coding: utf-8 + +module Treasury + module Fields + module Delayed + protected + + # Internal: Отмена отложенных задач по приращению + # + # Returns Integer количество отмененных задач + def cancel_delayed_increments + Resque.remove_delayed_selection(Treasury::DelayedIncrementJob) do |args| + args[0]['field_class'] == field_model.field_class + end + end + end + end +end diff --git a/lib/treasury/fields/product/base.rb b/lib/treasury/fields/product/base.rb index 131c202..6be8db2 100644 --- a/lib/treasury/fields/product/base.rb +++ b/lib/treasury/fields/product/base.rb @@ -3,7 +3,53 @@ module Treasury module Fields module Product - class Base < ::CoreDenormalization::Fields::Product::Base + # + # Базовый класс - Поле системы денормализации, для полей на основе товара. + # + class Base < Treasury::Fields::Base + BATCH_SIZE = 50_000 + + protected + + # Protected: Инициализирует параметры поля. + # + # Returns nothing. + + def init_params + super + self.batch_size = BATCH_SIZE + end + + # Protected: Возвращает идентификатор товара, переданного как объект в параметрах. + # + # params - Hash параметров: + # :object - String/Numeric/::Product или Hash, содержащий элемент + # :product или :product_id, указанных типов. + # + # Returns Numeric. + + def self.extract_object(params) + product = params.fetch(:object) + product = product.fetch(:product) || product.fetch(:product_id) if product.is_a?(Hash) + + case product + when ::Numeric + product + when ::String + product.to_i + else + if product && product.respond_to?(:id) + product.id + else + raise ArgumentError, "Product instance or Numeric/String product id expected!', #{params.inspect}" + end + end + end + + class << self + alias :extract_product :extract_object + alias :accessing_product :accessing_object + end end end end diff --git a/lib/treasury/fields/user/base.rb b/lib/treasury/fields/user/base.rb index fa4cbb5..253cafb 100644 --- a/lib/treasury/fields/user/base.rb +++ b/lib/treasury/fields/user/base.rb @@ -3,7 +3,56 @@ module Treasury module Fields module User - class Base < ::CoreDenormalization::Fields::User::Base + # + # Базовый класс - Поле системы денормализации, для полей на основе пользователя. + # + class Base < Treasury::Fields::Base + BATCH_SIZE = 25_000 + + protected + + # Protected: Инициализирует параметры поля. + # + # Returns nothing. + + def init_params + super + self.batch_size = BATCH_SIZE + end + + # Protected: Возвращает идентификатор пользователя, переданного как объект в параметрах. + # + # params - Hash параметров: + # :object - String/Numeric/::User или Hash, содержащий элемент + # :user или :user_id, указанных типов. + # + # Returns Numeric. + + def self.extract_object(params) + user = params[:object] + user = user[:user] || user[:user_id] if user.is_a?(Hash) + + case user + when ::Numeric + user + when ::String + user.to_i + else + if user && user.respond_to?(:id) + user.id + else + raise ArgumentError, "User instance or Numeric/String user id expected!', #{params.inspect}" + end + end + end + + class << self + alias :extract_user :extract_object + end + + class << self + alias :accessing_user :accessing_object + end end end end diff --git a/lib/treasury/logging.rb b/lib/treasury/logging.rb new file mode 100644 index 0000000..6874eba --- /dev/null +++ b/lib/treasury/logging.rb @@ -0,0 +1,16 @@ +# coding: utf-8 +require 'class_logger' + +module Treasury + module Logging + extend ActiveSupport::Concern + + included do + def self.logger_after_init + logger.blank_row + end + + include ::ClassLogger + end + end +end diff --git a/lib/treasury/migration/new_field.rb b/lib/treasury/migration/new_field.rb index ca18fac..41d15e8 100644 --- a/lib/treasury/migration/new_field.rb +++ b/lib/treasury/migration/new_field.rb @@ -50,8 +50,8 @@ def down private def create_denormalization_field - Denormalization::Models::Field.transaction do - field_record = Denormalization::Models::Field.create! do |f| + Treasury::Models::Field.transaction do + field_record = Treasury::Models::Field.create! do |f| f.title = field[:title] || field[:class].underscore.tr('/', '_') f.group = group f.field_class = field[:class] @@ -67,14 +67,14 @@ def create_denormalization_field f.oid = 0 end - queue = Denormalization::Models::Queue.create! do |q| + queue = Treasury::Models::Queue.create! do |q| q.name = consumer_name q.table_name = processor[:table_name] q.db_link_class = processor[:db_link_class] q.trigger_code = q.generate_trigger(processor[:trigger]) end - Denormalization::Models::Processor.create! do |f| + Treasury::Models::Processor.create! do |f| f.field = field_record f.queue = queue f.processor_class = processor[:class] @@ -88,16 +88,16 @@ def create_denormalization_field def delete_denormalization_field return if Rails.env.test? - Denormalization::Models::Processor.find_by_consumer_name(consumer_name).try(:destroy) - Denormalization::Models::Queue.find_by_name(consumer_name).try(:destroy) - Denormalization::Models::Field.find_by_field_class(field[:class]).try(:destroy) + Treasury::Models::Processor.find_by_consumer_name(consumer_name).try(:destroy) + Treasury::Models::Queue.find_by_name(consumer_name).try(:destroy) + Treasury::Models::Field.find_by_field_class(field[:class]).try(:destroy) end def find_or_create_worker worker = respond_to?(:worker) ? worker : :common - Denormalization::Models::Worker.find_by_name(worker) || - Denormalization::Models::Worker.create!(name: worker, active: true) + Treasury::Models::Worker.find_by_name(worker) || + Treasury::Models::Worker.create!(name: worker, active: true) end def consumer_name @@ -107,11 +107,11 @@ def consumer_name def storage(name) case name when :redis - 'CoreDenormalization::Storage::Redis::Multi' + 'Treasury::Storage::Redis::Multi' when :db - 'CoreDenormalization::Storage::PostgreSQL::Db' + 'Treasury::Storage::PostgreSQL::Db' when :pgq - 'CoreDenormalization::Storage::PostgreSQL::PgqProducer' + 'Treasury::Storage::PostgreSQL::PgqProducer' else raise 'Unexpected storage' end diff --git a/lib/treasury/pgq.rb b/lib/treasury/pgq.rb new file mode 100644 index 0000000..6b5dd58 --- /dev/null +++ b/lib/treasury/pgq.rb @@ -0,0 +1,105 @@ +module Treasury + module Pgq + # http://skytools.projects.postgresql.org/pgq/files/external-sql.html + # http://skytools.projects.postgresql.org/skytools-3.0/pgq/files/external-sql.html + # http://skytools.projects.postgresql.org/doc/pgq-sql.html + # https://github.com/markokr/skytools + + def pgq_create_queue(queue_name, conn) + conn.select_value("SELECT pgq.create_queue(#{connection.quote queue_name})").to_i + end + + def pgq_drop_queue(queue_name, conn) + conn.select_value("SELECT pgq.drop_queue(#{connection.quote queue_name})").to_i + end + + def pgq_insert_event(queue_name, ev_type, ev_data, extra1 = nil, extra2 = nil, extra3 = nil, extra4 = nil) + result = connection.select_value(<<-SQL.squish) + SELECT pgq.insert_event( + #{connection.quote queue_name}, + #{connection.quote ev_type}, + #{connection.quote ev_data}, + #{connection.quote extra1}, + #{connection.quote extra2}, + #{connection.quote extra3}, + #{connection.quote extra4} + ) + SQL + + result ? result.to_i : nil + end + + def pgq_register_consumer(queue_name, consumer_name, conn) + result = conn.select_value(<<-SQL.squish) + SELECT pgq.register_consumer(#{connection.quote queue_name}, #{connection.quote consumer_name}) + SQL + + result.to_i + end + + def pgq_unregister_consumer(queue_name, consumer_name, conn) + result = conn.select_value(<<-SQL.squish) + SELECT pgq.unregister_consumer(#{connection.quote queue_name}, #{connection.quote consumer_name}) + SQL + + result.to_i + end + + def pgq_next_batch(queue_name, consumer_name, conn) + result = conn.select_value(<<-SQL.squish) + SELECT pgq.next_batch(#{connection.quote queue_name}, #{connection.quote consumer_name}) + SQL + + result ? result.to_i : nil + end + + def pgq_get_batch_events(batch_id, conn) + conn.select_all("SELECT * FROM pgq.get_batch_events(#{batch_id})") + end + + def get_batch_events_by_cursor(batch_id, cursor_name, fetch_size, extra_where, conn) + conn.select_all(<<-SQL.squish) + SELECT * FROM pgq.get_batch_cursor( + #{batch_id}, + #{connection.quote(cursor_name)}, + #{fetch_size}, + #{connection.quote(extra_where)} + ) + SQL + end + + def pgq_event_failed(batch_id, event_id, reason, conn) + conn.select_value(sanitize_sql(["SELECT pgq.event_failed(?, ?, ?)", batch_id, event_id, reason])).to_i + end + + def pgq_event_retry(batch_id, event_id, retry_seconds, conn) + conn.select_value("SELECT pgq.event_retry(#{batch_id}, #{event_id}, #{retry_seconds})").to_i + end + + def pgq_finish_batch(batch_id, conn) + conn.select_value("SELECT pgq.finish_batch(#{batch_id})") + end + + def pgq_get_queue_info(queue_name, conn) + conn.select_all("SELECT pgq.get_queue_info(#{connection.quote queue_name})") + end + + def pgq_queue_exists?(queue_name, conn) + pgq_get_queue_info(queue_name, conn).present? + end + + def pgq_force_tick(queue_name, conn) + last_tick = conn.select_value sanitize_sql(["SELECT pgq.force_tick(:queue_name)", {queue_name: queue_name}]) + current_tick = conn.select_value sanitize_sql(["SELECT pgq.force_tick(:queue_name)", {queue_name: queue_name}]) + cnt = 0 + + while last_tick != current_tick and cnt < 100 + current_tick = conn.select_value sanitize_sql(["SELECT pgq.force_tick(:queue_name)", {queue_name: queue_name}]) + sleep 0.01 + cnt += 1 + end + + current_tick + end + end +end diff --git a/lib/treasury/pgq/consumer.rb b/lib/treasury/pgq/consumer.rb new file mode 100644 index 0000000..cf3d7e4 --- /dev/null +++ b/lib/treasury/pgq/consumer.rb @@ -0,0 +1,105 @@ +# coding: utf-8 + +module Treasury + module Pgq + class Consumer + attr_accessor :queue, :consumer, :connection + + def initialize(queue, consumer, connection) + self.queue = queue + self.consumer = consumer + self.connection = connection + end + + def self.quote(text) + ActiveRecord::Base.connection.quote(text) + end + + def self.get_consumer_info(connection = ActiveRecord::Base.connection) + connection.select_all("select c.*, trunc(extract(minutes from lag)*60 + extract(seconds from lag)) seconds_lag from pgq.get_consumer_info() c") + end + + def self.consumer_exists?(queue_name, consumer, connection = ActiveRecord::Base.connection) + connection.select_all("select * from pgq.get_consumer_info() WHERE queue_name = #{quote(queue_name)} AND consumer_name = #{quote(consumer)}").present? + end + + def self.failed_event_retry(queue_name, consumer, event_id, connection = ActiveRecord::Base.connection) + connection.select_value( + "select * from pgq.failed_event_retry(#{self.quote queue_name}, #{self.quote consumer},#{event_id.to_i})") + end + + def self.failed_event_delete(queue_name, consumer, event_id, connection = ActiveRecord::Base.connection) + connection.select_value( + "select * from pgq.failed_event_delete(#{self.quote queue_name}, #{self.quote consumer},#{event_id.to_i})") + end + + def self.failed_event_count(queue_name, consumer, connection = ActiveRecord::Base.connection) + connection.select_value("select * from pgq.failed_event_count(#{self.quote queue_name}, #{self.quote consumer})") + end + + def self.failed_event_list(queue_name, consumer, cnt = nil, offset = nil, connection = ActiveRecord::Base.connection) + off = '' + off = ",#{cnt.to_i},#{offset.to_i}" if cnt.present? + connection.select_all("select * from pgq.failed_event_list(#{self.quote queue_name}, #{self.quote consumer} #{off}) order by ev_id desc") + end + + def get_batch_events + @batch_id = get_next_batch + return nil if @batch_id.nil? + ActiveRecord::Base.pgq_get_batch_events(@batch_id, connection) + end + + def get_batch_events_by_cursor(batch_id, cursor_name, fetch_size = 1000, extra_where = nil) + ActiveRecord::Base.get_batch_events_by_cursor(batch_id, cursor_name, fetch_size, extra_where, connection) + end + + def get_next_batch + ActiveRecord::Base.pgq_next_batch(queue, consumer, connection) + rescue ActiveRecord::StatementInvalid => e + raise unless e.message =~ /Not subscriber to queue/ + raise Errors::QueueOrSubscriberNotFoundError.new(e) + end + + def finish_batch + ActiveRecord::Base.pgq_finish_batch(@batch_id, connection) + end + + def event_failed(event_id, reason) + ActiveRecord::Base.pgq_event_failed(@batch_id, event_id, reason, connection) + end + + def event_retry(event_id, retry_seconds) + ActiveRecord::Base.pgq_event_retry(@batch_id, event_id, retry_seconds, connection) + end + + def process + events = get_batch_events + return if !events + events.each do |event| + perform_event(prepare_event(event)) + end + + finish_batch + true + end + + alias perform_batch process + + def perform_event(event) + end + + def prepare_event(event) + Event.new event + end + + #def add_event data + # self.class.add_event data + #end + # + #def self.add_event data + # ActiveRecord::Base.pgq_insert_event(self.const_get('QUEUE_NAME'), self.const_get('TYPE'), data.to_yaml) + #end + + end + end +end diff --git a/lib/treasury/pgq/errors.rb b/lib/treasury/pgq/errors.rb new file mode 100644 index 0000000..05b8234 --- /dev/null +++ b/lib/treasury/pgq/errors.rb @@ -0,0 +1,28 @@ +# coding: utf-8 + +module Treasury + module Pgq + module Errors + class PgqError < StandardError + attr_accessor :inner_exception + + def initialize(inner_exception, message = nil) + super(message) + @inner_exception = inner_exception + end + + def message + "#{super}\r\n#{@inner_exception.try(:message)}" + end + end + + class QueueOrSubscriberNotFoundError < PgqError + DEFAULT_MESSAGE = "Не найдена очередь или подписчик.".freeze + + def initialize(inner_exception, message = DEFAULT_MESSAGE) + super(inner_exception, message) + end + end + end + end +end diff --git a/lib/treasury/pgq/event.rb b/lib/treasury/pgq/event.rb new file mode 100644 index 0000000..a72166d --- /dev/null +++ b/lib/treasury/pgq/event.rb @@ -0,0 +1,88 @@ +# coding: utf-8 + +module Treasury + module Pgq + class Event + TYPE_INSERT = 'I'.freeze + TYPE_UPDATE = 'U'.freeze + TYPE_DELETE = 'D'.freeze + + attr_accessor :id, :type, :birth_time, :txid, :ev_data, :extra1, :extra2, :extra3, :extra4 + + def initialize(pgq_tuple) + assign(pgq_tuple) if pgq_tuple + end + + def assign(pgq_tuple) + @id = pgq_tuple['ev_id'].to_i + @type = pgq_tuple['ev_type'].split(':').first + @birth_time = pgq_tuple['ev_time'].try(:to_time) + @txid = pgq_tuple['ev_txid'] + @ev_data = pgq_tuple['ev_data'] + @extra1 = pgq_tuple['ev_extra1'] + @extra2 = pgq_tuple['ev_extra2'] + @extra3 = pgq_tuple['ev_extra3'] + @extra4 = pgq_tuple['ev_extra4'] + + @data = nil + @prev_data = nil + @raw_data = nil + @raw_prev_data = nil + @user_data = nil + end + + def raw_data + # мой простой вариант, быстрее ~ в 3 раза, но не делает unescape и normalize_params + @raw_data ||= @data || simple_parse_query(@ev_data) + end + + def raw_prev_data + # мой простой вариант, быстрее ~ в 3 раза, но не делает unescape и normalize_params + @raw_prev_data ||= @prev_data || simple_parse_query(@extra2) + end + + def data + # parse_nested_query - очень узкое место, без неё скорость возростает в 2,5 раза + @data ||= HashWithIndifferentAccess.new(Rack::Utils.parse_nested_query(@ev_data)) + end + + def prev_data + @prev_data ||= HashWithIndifferentAccess.new(Rack::Utils.parse_nested_query(@extra2)) + end + + def user_data + @user_data ||= HashWithIndifferentAccess.new(Rack::Utils.parse_nested_query(@extra3)) + end + + def insert? + @type == TYPE_INSERT + end + + def update? + @type == TYPE_UPDATE + end + + def delete? + @type == TYPE_DELETE + end + + def data_changed? + @ev_data != @extra2 + end + + def no_data_changed? + !data_changed? + end + + protected + + def simple_parse_query(query) + return {} if query.nil? + query.split('&').inject(HashWithIndifferentAccess.new) do |result, item| + k, v = item.split('=') + result.merge!(k => v) + end + end + end + end +end diff --git a/lib/treasury/pgq/snapshot.rb b/lib/treasury/pgq/snapshot.rb new file mode 100644 index 0000000..dfc0e79 --- /dev/null +++ b/lib/treasury/pgq/snapshot.rb @@ -0,0 +1,53 @@ +# coding: utf-8 + +module Treasury + module Pgq + class Snapshot + # represents a PostgreSQL snapshot. + # http://www.postgresql.org/docs/8.3/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS + # http://skytools.projects.postgresql.org/txid/functions-txid.html + + # example: + # >>> sn = Snapshot.new('11:20:11,12,15') + # >>> sn.contains?(9) + # True + # >>> sn.contains?(11) + # False + # >>> sn.contains?(17) + # True + # >>> sn.contains?(20) + # False + + attr_reader :as_string + attr_reader :xmin + attr_reader :xmax + attr_reader :txid_list + + def initialize(string_snapshot) + # create snapshot from string + @as_string = string_snapshot + parts = string_snapshot.split(':') + raise 'Unknown format for snapshot' unless (2..3).include?(parts.size) + @xmin = parts[0].to_i + @xmax = parts[1].to_i + @txid_list = [] + return if parts[2].blank? + @txid_list = parts[2].split(',').map(&:to_i) + end + + def contains?(txid) + # is txid visible in snapshot + txid = txid.to_i + if txid < xmin + return true + elsif txid >= xmax + return false + elsif txid_list.include?(txid) + return false + end + + true + end + end + end +end diff --git a/lib/treasury/processors/base.rb b/lib/treasury/processors/base.rb index 060c51e..b2986ce 100644 --- a/lib/treasury/processors/base.rb +++ b/lib/treasury/processors/base.rb @@ -2,15 +2,79 @@ module Treasury module Processors - class Base < ::CoreDenormalization::Processors::Base + class Base < Treasury::Pgq::Consumer + DEFAULT_FETCH_SIZE = 5000 + + attr_reader :logger + attr_reader :processor_info + attr_reader :consumer_name + attr_reader :queue_name + attr_reader :data + attr_reader :params + attr_reader :event attr_accessor :object - + + def initialize(processor_info, logger = Rails.logger) + @processor_info = processor_info + @logger = logger + @event = Treasury::Pgq::Event.new(nil) + @snapshot = Treasury::Pgq::Snapshot.new(field.field_model.snapshot_id) + @cursor_name = "denorm_proc_#{processor_info.id}" + + init_params + super(@queue_name, @consumer_name, work_connection) + end + + def process + events_processed = 0 + rows_written = 0 + @changed_keys = [] + + get_batch + return {:events_processed => 0, :rows_written => 0} unless @batch_id + + init_storage + + main_connection.transaction do + work_connection.transaction do + before_batch_processing + + start_storage_transaction + + reset_buffer + + get_batch_events do |events| + process_events_batch(events) + events_processed += events.size + end + + write_data + + rows_written = @data.size + @changed_keys = @data.keys + + after_batch_processing + + commit_storage_transaction + + finish_batch + end + end + + data_changed + + {:events_processed => events_processed, :rows_written => rows_written} + rescue StandardError, NotImplementedError + rollback_storage_transaction + raise + end + def current_value(field_name = nil) object_value(@object, field_name) end def object_value(l_object, field_name = nil) - value = if @data.key?(l_object) + value = if @data.key?(l_object) && @data[l_object].key?(field_name || field.first_field) @data[l_object][field_name || field.first_field] else field.raw_value(l_object, field_name) @@ -30,6 +94,307 @@ def result_row(value) def no_action nil end + + protected + + def init_params + @params = @processor_info.params || HashWithIndifferentAccess.new + @queue_name = @processor_info.queue.pgq_queue_name + @consumer_name = @processor_info.consumer_name + @fetch_size = DEFAULT_FETCH_SIZE + end + + def init_event_params + @object = @event.raw_data[:id] + end + + def process_event + case @event.type + when Pgq::Event::TYPE_INSERT + process_insert + when Pgq::Event::TYPE_UPDATE + process_update if @event.data_changed? + when Pgq::Event::TYPE_DELETE + process_delete + else + raise Errors::UnknownEventTypeError + end + end + + def process_insert + raise NotImplementedError + end + + def process_update + raise NotImplementedError + end + + def process_delete + raise NotImplementedError + end + + def after_process_event + end + + def current_value_as_integer(field_name = nil) + current_value(field_name).to_i + end + + def incremented_current_value(field_name = nil, by = 1) + current_value_as_integer(field_name) + by + end + + def decremented_current_value(field_name = nil, by = 1) + current_value_as_integer(field_name) - by + end + + # Protected: Зануляет поля расчитываемые обработчиком. + # + # Returns Hash. + # + def nullify_current_value + result_row(nil) + end + + # Protected: Помечает строку для удаления. + # + # Returns Hash. + # + def delete_current_row + {@object => nil} + end + + # Protected: Удаляет поля расчитываемые обработчиком. + # + # В зависимости от того является ли обработчик ведущим, либо удаляет + # строку из хранилища, либо зануляет значение. + # + # Returns Hash. + # + def delete_current_value + master? ? delete_current_row : nullify_current_value + end + + # Protected: Возвращает признак - является ли обработчик, ведущим. + # + # Returns Boolean. + # + def master? + params.fetch(:master, false) + end + + # колбеки, для перекрытия в наследниках + + def before_batch_processing; end + + def after_batch_processing; end + + def field + @field ||= Treasury::Fields::Base.create_by_class(field_class, processor_info.field) + end + + # Protected: Отложить событие в очередь на последующую обработку. + # + # Returns no_action. + # + def event_retry(retry_seconds) + super(@event.id, retry_seconds) + logger.warn "Событие отложено! (#{@event.inspect})" + no_action + end + + # Protected: Регистрирует событие в журнале. + # + # Returns no_action. + # + def log_event(params) + params = { + :consumer => consumer_name, + :event => @event + }.merge!(params) + + Treasury::Services::EventsLogger.add(params) + end + + # Protected: Рабочее соединение с БД для данной очереди. + # + # В рамках этого соединения производится обработка событий конкретного процессора. + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def work_connection + @work_connection ||= processor_info.queue.work_connection + end + + # Protected: Основное соединение с БД. + # + # В рамках этого соединения производятся общие действия (изменения метаданных). + # + # Returns ActiveRecord::ConnectionAdapters::AbstractAdapter. + # + def main_connection + ActiveRecord::Base.connection + end + + # Protected: Возвращает идентификатор источника событий (по сути уникальный идентификатор БД) + # + # Returns String. + # + def source_id + Digest::MD5.hexdigest(db_link_class) if db_link_class.present? + end + + def db_link_class + @db_link_class ||= processor_info.queue.db_link_class + end + + private + + def interesting_event? + # фильтруем события, которые были поставлены в очередь, + # но были видны и обработаны при иницилизации + # TODO: подумать как не фильтровать все время, в лондисте есть механизм, я видел + !@snapshot.contains?(@event.txid) + end + + def write_data + return if @data.empty? + storages.each do |storage| + storage.bulk_write(@data) + end + end + + def start_storage_transaction + storages.each(&:start_transaction) + end + + def commit_storage_transaction + storages.each do |storage| + storage.add_batch_to_processed_list(@batch_id) + storage.commit_transaction + end + end + + def rollback_storage_transaction + storages.each(&:rollback_transaction) + end + + def reset_buffer + @data = HashWithIndifferentAccess.new + end + + def field_class + @field_class ||= @processor_info.field.field_class + end + + # Internal: метод возврашающий хранилища используемые данным процессором, кешируется + # Важно! Если при определении процессора указан массив :params => {..., :storage_ids => [...]} + # данный процессор будет работать только с указанными по ID хранилищами. + # + # Returns Array of Storage + def storages + @storages ||= if @params.key?(:storage_ids) + field.storages_by_ids(@params[:storage_ids]) + else + field.storages + end + end + + def get_batch + @batch_id = get_next_batch + end + + def process_events_batch(events) + # logger.debug "Событий в батче: #{events.size}" if events.size > 0 + events.each { |ev| internal_process_event(ev) } + end + + def internal_process_event(ev) + # @logger.info "event: #{ev.inspect}" + @event.assign(ev) + log_event(:message => 'input') + + unless interesting_event? + log_event(:message => 'not interesting. passed.', :payload => @snapshot.as_string) + return + end + + init_event_params + result = process_event + + log_event(:message => 'processing result', :payload => result) + @data.deep_merge!(result) if result.present? + after_process_event + # @logger.info "result_row: #{result.inspect}" if result.present? + end + + def get_batch_events + events = get_batch_events_by_cursor(@batch_id, @cursor_name, @fetch_size) + + return events if events.empty? + + if batch_already_processed? + log_event(:message => 'batch already processed', :payload => @batch_id) + logger.warn "Батч #{@batch_id} уже был ранее обработан! События пропущены." + return [] + end + + yield events + + options = { + name: @cursor_name, + connection: work_connection, + fetch_size: @fetch_size + } + + PgTools::Cursor.each_batch(options) { |events| yield events } + end + + # метод проверяет целостность данных в хранилищах + # обеспечивает защиту от повторного выполнения батча + # при нарушение целостности данных, генерирует исключение + # нарушение целостности данных возможно когда используется больше одного хранилища, + # если в одно хранилище данные были записаны, а в другое нет + # return boolean + # true - батч уже был ранее обработан и должен быть пропущен + # false - батч, не был ранее обработан и должен быть обработан + def batch_already_processed? + return false if storages.blank? + + # проверяем тип первого (основного хранилища) + if storages.first.use_own_connection? + # если используется своя сессия, проверяем обработан ли батч + # если нет, выходим, ведем обычную обработку + return if storages.first.batch_no_processed?(@batch_id) + # если обработан, то выбираем хранилища использующие свою сессию и не обработавшие батч + # использующие основную сессию + bad_storages = storages.select { |s| s.use_own_connection? && s.batch_no_processed?(@batch_id) } + bad_storages += storages.select(&:use_processor_connection?) + # если таких, нет, то батч считается обработаннм и пропускается + return true if bad_storages.blank? + # если есть, то требуется переиницилизация поля + else + # если используется основная сессия, то: + # проверяем, есть ли хранилища, со своей сессией, где обработан батч + # если нет, выходим, ведем обычную обработку + return unless storages.any? { |s| s.use_own_connection? && s.batch_already_processed?(@batch_id) } + # если есть, то требуется переиницилизация поля + end + + raise Errors::InconsistencyDataError + end + + def data_changed + return unless @changed_keys.present? + field.send(:data_changed, @changed_keys) + rescue => e + logger.error "Ошибка при вызове колбека data_changed:" + logger.error "#{e.message}\n\n #{e.backtrace.join("\n")}" + end + + def init_storage + storages.each { |storage| storage.source = self } + end end end end diff --git a/lib/treasury/processors/company/base.rb b/lib/treasury/processors/company/base.rb index 5971a78..1d66e65 100644 --- a/lib/treasury/processors/company/base.rb +++ b/lib/treasury/processors/company/base.rb @@ -3,7 +3,20 @@ module Treasury module Processors module Company - class Base < ::CoreDenormalization::Processors::Company::Base + class Base < ::Treasury::Processors::Base + alias :company_id= :object= + alias :company_id :object + + protected + + def init_event_params + self.company_id = extract_company.to_i.nonzero? + raise ArgumentError, "Company ID expected to be Integer, #{@event.inspect}" unless company_id + end + + def extract_company + @event.raw_data.key?(:company_id) ? @event.raw_data[:company_id] : @event.raw_data[:id] + end end end end diff --git a/lib/treasury/processors/delayed.rb b/lib/treasury/processors/delayed.rb new file mode 100644 index 0000000..91f7bda --- /dev/null +++ b/lib/treasury/processors/delayed.rb @@ -0,0 +1,88 @@ +# coding: utf-8 + +module Treasury + module Processors + module Delayed + # Отложенное приращение текущего значения поля + # + # field_name - String + # seconds - Integer + # + # Returns nothing + def delayed_increment_current_value(field_name, seconds) + delayed_change_current_value(field_name, seconds, 1) + end + + # Отложенное уменьшение текущего значения поля + # + # field_name - String + # seconds - Integer + # + # Returns nothing + def delayed_decrement_current_value(field_name, seconds) + delayed_change_current_value(field_name, seconds, -1) + end + + # Отмена отложенного приращения значения + # Note: возвращает false если отложенной задачи не найдено + # + # field_name - String + # + # Returns boolean + def cancel_delayed_increment(field_name) + cancel_delayed_change(field_name, 1) + end + + # Отмена отложенного уменьшения значения + # Note: возвращает false если отложенной задачи не найдено + # + # field_name - String + # + # Returns boolean + def cancel_delayed_decrement(field_name) + cancel_delayed_change(field_name, -1) + end + + private + + # Internal: Отложенное изменение текущего значения поля на указанную величину + # + # field_name - String + # seconds - Integer + # by - Integer величина, на которую нужно изменить + # + # Returns nothing + def delayed_change_current_value(field_name, seconds, by) + job_params = delayed_increment_job_params(field_name, by) + Resque.enqueue_in(seconds, Treasury::DelayedIncrementJob, job_params) + + no_action + end + + # Internal: Отмена отложенного изменения значения + # Note: возвращает false если отложенной задачи не найдено + # + # field_name - String + # + # Returns boolean + def cancel_delayed_change(field_name, by) + job_params = delayed_increment_job_params(field_name, by) + removed = Resque.remove_delayed(Treasury::DelayedIncrementJob, job_params) + + !removed.zero? + end + + def delayed_increment_job_params(field_name, by) + delayed_job_params.merge! field_name: field_name, by: by + end + + def delayed_job_params + { + id: event.data.fetch(:id), + object: @object, + field_class: field_class + } + end + end + end +end diff --git a/lib/treasury/processors/errors.rb b/lib/treasury/processors/errors.rb new file mode 100644 index 0000000..016b421 --- /dev/null +++ b/lib/treasury/processors/errors.rb @@ -0,0 +1,11 @@ +# coding: utf-8 + +module Treasury + module Processors + module Errors + class ProcessorError < StandardError; end + class UnknownEventTypeError < ProcessorError; end + class InconsistencyDataError < ProcessorError; end + end + end +end diff --git a/lib/treasury/processors/optimized_translator.rb b/lib/treasury/processors/optimized_translator.rb index 68e17a4..2881ae6 100644 --- a/lib/treasury/processors/optimized_translator.rb +++ b/lib/treasury/processors/optimized_translator.rb @@ -1,7 +1,34 @@ module Treasury module Processors + # процессор - оптимизированный транслятор данных + # транслирует данные поступающие в очередь - в хранилище + # отличие от обычного транслятора: + # в результирующем хэше, возвращает только измененные поля + # из-за этого, его нельзя использовать, с некоторыми типами хранилищ module OptimizedTranslator - include ::CoreDenormalization::Processors::OptimizedTranslator + include Treasury::Processors::Translator + + protected + + def process_update + is_prev_version_stored = prev_version_stored? + is_data_changed = false + + fields_result = fields_map.inject(result_hash) do |result, field| + source = field[:source] + target = field[:target] + if (@event.data[source] == @event.prev_data[source]) && is_prev_version_stored + result + else + is_data_changed = true + result.merge!(target => @event.data[source]) + end + end + + return nullify_current_value unless need_translate?(@event.data) + return no_action unless is_data_changed + result_row fields_result + end end end end diff --git a/lib/treasury/processors/product.rb b/lib/treasury/processors/product.rb new file mode 100644 index 0000000..568d512 --- /dev/null +++ b/lib/treasury/processors/product.rb @@ -0,0 +1,24 @@ +# coding: utf-8 +module Treasury + module Processors + module Product + extend ActiveSupport::Concern + + included do + alias_method :product_id=, :object= + alias_method :product_id, :object + end + + protected + + def init_event_params + self.product_id = extract_product + raise ArgumentError, "Product ID expected to be Integer, #{@event.inspect}" unless product_id + end + + def extract_product + @event.raw_data[:product_id] || @event.fetch(:id) + end + end + end +end diff --git a/lib/treasury/processors/translator.rb b/lib/treasury/processors/translator.rb index daad485..0fee0cb 100644 --- a/lib/treasury/processors/translator.rb +++ b/lib/treasury/processors/translator.rb @@ -1,7 +1,90 @@ +# coding: utf-8 + module Treasury module Processors + # процессор - транслятор данных + # транслирует данные поступающие в очередь - в хранилище + # параметры: + # :fields_map => [{:source => source_field, :target => target_field}] + # - карта полей, массив хэшей, указывающий какое поле источника куда положить в хранилище + # По умолчанию: все поля. + # при изменении данных, хотя бы в одном поле, возвращает полный хэш значений + # Позволяет, в потомке с помощью перекрытия метода need_translate?, + # задать какие данные нужны, а какие нет. module Translator - include ::CoreDenormalization::Processors::Translator + protected + + # Protected: Нужно ли транслировать в хранилище текущую строку. + # + # Позволяет задать произвольное условие, которое определяет, + # какие строки нужно хранить, а какие нет. + # Строки, для которых данный метод вернул false, не будут записаны + # в хранилище при insert и будут удалены при update событиях. + # + # data - Hash - данные события + # + # Returns Boolean. + def need_translate?(data) + true + end + + def process_insert + return no_action unless need_translate?(@event.data) + + result_row( + fields_map.inject(result_hash) do |result, field| + result.merge!(field[:target] => @event.data.fetch(field[:source])) + end + ) + end + + def process_update + is_data_changed = false + result = params[:fields_map].inject(result_hash) do |result, field| + source = field[:source] + target = field[:target] + result[target] = @event.data[source] + unless @event.data[source] == @event.prev_data[source] + is_data_changed = true + end + + result + end + + return nullify_current_value unless need_translate?(@event.data) + return no_action unless is_data_changed || !prev_version_stored? + result_row result + end + + def process_delete + delete_current_value + end + + def nullify_current_value + result_row( + params[:fields_map].inject(result_hash) do |result, field| + result.merge!({field[:target] => nil}) + end + ) + end + + def prev_version_stored? + need_translate?(@event.prev_data) + end + + private + + def fields_map + @fields_map ||= params[:fields_map] || default_fields_map + end + + def default_fields_map + field.send(:fields_for_reset).map { |field| {:source => field, :target => field} } + end + + def result_hash + HashWithIndifferentAccess.new + end end end end diff --git a/lib/treasury/processors/user/base.rb b/lib/treasury/processors/user/base.rb index 46aff5e..a9c89a3 100644 --- a/lib/treasury/processors/user/base.rb +++ b/lib/treasury/processors/user/base.rb @@ -3,7 +3,7 @@ module Treasury module Processors module User - class Base < Treasury::Processors::Base + class Base < ::Treasury::Processors::Base alias :user_id= :object= alias :user_id :object diff --git a/lib/treasury/services/events_logger.rb b/lib/treasury/services/events_logger.rb new file mode 100644 index 0000000..13152db --- /dev/null +++ b/lib/treasury/services/events_logger.rb @@ -0,0 +1,198 @@ +# coding: utf-8 +require 'class_logger' + +module Treasury + module Services + # Логгер обрабатываемых событий. + # Позволяет отлаживать обработчики в тех случаях, когда данные считаются не верно. + # Пишет в Redis. Умеет выгружать в БД. + # + class EventsLogger + include ::ClassLogger + + LOGGER_FILE_NAME = "#{ROOT_LOGGER_DIR}/events_logger".freeze + INROW_DELIMITER = '{{!cslirs!}}'.freeze + INROW_EMPTY = 'nil'.freeze + ROWS_PER_LOOP = 5000 + + attr_accessor :params + + def self.add(params) + @instance ||= self.new + @instance.params = params + @instance.add + end + + def add + return unless need_logging? + + time = Time.now.utc + key = events_list_key(time.to_date) + value = [ + time, + params[:consumer], + params[:event].id, + params[:event].type, + params[:event].time, + params[:event].txid, + params[:event].ev_data, + params[:event].extra1, + params[:event].data, + params[:event].prev_data, + params[:event].data_changed?.to_s, + params[:message] || INROW_EMPTY, + params[:payload].inspect || INROW_EMPTY, + suspected_event?.to_s + ].join(INROW_DELIMITER) + + redis.rpush(key, value) + redis.sadd(dates_set_key, key) + + logger.fatal(value) if suspected_event? + end + + def process(date, clear_table = false, delete_from_redis = true) + raise ':date param is not a date/time' unless date.is_a?(Date) || date.is_a?(Time) + + key = events_list_key(date) + unless redis.exists(key) + logger.warn("Log rows for date [#{date}] not found") + return + end + + rows_count = redis.llen(key).to_i + logger.info("[#{rows_count}] rows found for date [#{date}]") + return if rows_count.zero? + + processed = 0 + model_class.transaction do + offset = 0 + limit = ROWS_PER_LOOP + + model_class.clear(date) if clear_table + while rows = redis.lrange(key, offset, offset + limit - 1) + processed_rows = {:rows => []} + for row in rows do + processed_row = process_row(row) + processed_rows[:fields] ||= processed_row.keys + processed_rows[:rows] << quote(processed_row.values).join(', ') + end + + insert_into_table(processed_rows[:fields], processed_rows[:rows]) + processed += processed_rows[:rows].size + + logger.info("[#{processed}/#{rows_count}] rows processed for date [#{date}]") + break if rows.length < limit + offset += limit + end + end + + delete_events(date) if delete_from_redis + rescue => e + logger.fatal "#{e.message}\n\n #{e.backtrace.join("\n")}" + ErrorMailer.custom_error(text: "Ошибка при переносе лога обработанных событий Treasury::EventsLogger!", + message: e.inspect, + backtrace: e.backtrace).deliver + raise + end + + def delete_events(date) + raise ':date param is not a date/time' if !date.is_a?(Date) && !date.is_a?(Time) + + key = events_list_key(date) + logger.info("Removing events key [#{key}] from Redis") + logger.error("Redis key [#{key}] remove fail") unless redis.del(key) + logger.info("Removing member [#{key}] of set [#{dates_set_key}] from Redis") + logger.error("Redis member [#{key}] of set [#{dates_set_key}] remove fail") unless redis.srem(dates_set_key, key) + end + + def dates_list + result = {} + dates = redis.smembers(dates_set_key) + return result if dates.blank? + dates.each do |date_key| + if redis.exists(date_key) + result[date_key] = redis.llen(date_key).to_i + else + result[date_key] = 'not exists' + end + end + result + end + + protected + + def need_logging? + false + end + + def suspected_event? + false + end + + def events_list_key(date) + "#{root_key}:eventslog:#{date.strftime('%Y:%m:%d')}" + end + + def dates_set_key + "#{root_key}:eventslog:dates:list" + end + + def root_key + "#{Treasury::ROOT_REDIS_KEY}" + end + + def redis + Treasury.configuration.redis + end + + def process_row(row) + data = row.split(INROW_DELIMITER) + data.map! { |value| value.strip.eql?(INROW_EMPTY) ? nil : value } + + { + processed_at: data[0], + consumer: data[1], + event_id: data[2], + event_type: data[3], + event_time: data[4], + event_txid: data[5], + event_ev_data: data[6], + event_extra1: data[7], + event_data: data[8], + event_prev_data: data[9], + event_data_changed: data[10].to_b, + message: data[11], + payload: data[12], + suspected: data[13].to_b + } + end + + def insert_into_table(fields, data) + connection.execute(<<-SQL) + INSERT INTO #{model_class.quoted_table_name} + (#{fields.join(', ')}) + VALUES + (#{data.join('),(')}); + SQL + end + + def quote(data) + return data unless data.is_a?(Array) + data.map { |value| connection.quote(value) } + end + + def connection + model_class.connection + end + + def model_class + Treasury::Models::EventsLog + end + + def self.logger_default_file_name + LOGGER_FILE_NAME + end + end + end +end diff --git a/lib/treasury/session.rb b/lib/treasury/session.rb new file mode 100644 index 0000000..6eff44f --- /dev/null +++ b/lib/treasury/session.rb @@ -0,0 +1,42 @@ +# coding: utf-8 + +module Treasury + module Session + extend ActiveSupport::Concern + + module ClassMethods + def pid + @@pid ||= Process.pid + end + + def process_is_alive?(pid) + pid && process_exists?(pid) + end + + def process_is_dead?(pid) + !process_is_alive?(pid) + end + + def process_exists?(pid) + Process.getpgid(pid) + true + rescue Errno::ESRCH + false + end + end + + module InstanceMethods + def pid + self.class.pid + end + + def process_is_alive?(pid) + self.class.process_is_alive?(pid) + end + + def process_is_dead?(pid) + self.class.process_is_dead?(pid) + end + end + end +end diff --git a/lib/treasury/storage/base.rb b/lib/treasury/storage/base.rb new file mode 100644 index 0000000..7adaab9 --- /dev/null +++ b/lib/treasury/storage/base.rb @@ -0,0 +1,146 @@ +# coding: utf-8 +require 'callbacks_rb' + +module Treasury + module Storage + class Base + class_attribute :default_reset_strategy + self.default_reset_strategy = :reset + + attr_accessor :params + attr_reader :source + + def initialize(params) + @callbacks = {} + @params = default_params.deep_merge(params || {}) + end + + def bulk_write(data) + raise NotImplementedError + end + + def transaction_bulk_write(data) + raise NotImplementedError + end + + def read(object, field) + raise NotImplementedError + end + + # Public: Выполняет сброс данных хранилища + # + # object - Array or nil - Список идентификаторов объектов для сброса. + # Если nil, то выполняется сброс всех объектов. + # + # fields - Array - Список полей для сброса. + # + # Returns nothing. + def reset_data(objects, fields = nil) + raise NotImplementedError + end + + def start_transaction + @transaction_started = true + end + + def commit_transaction + @transaction_started = false + end + + def rollback_transaction + @transaction_started = false + end + + def transaction_started? + @transaction_started + end + + def transaction(&block) + start_transaction + begin + yield + commit_transaction + rescue + rollback_transaction + raise + end + end + + def id + @params[:id] + end + + # использует ли хранилище собственное, не зависимое от основного, соединение + # от этого зависит управление транзакцией записи данных в хранилище + def use_own_connection? + raise NotImplementedError + end + + def use_processor_connection? + !use_own_connection? + end + + # выполняет блокировку хранилища, перед иницилизацией, если это необходимо + def lock + end + + def source=(source) + @source = source + reset_source + end + + protected + + def default_id + raise NotImplementedError + end + + # Protected: Добалвяет префикс хранения полю + # + # field - String/Symbol - имя поля + # + # Returns string. + + def prepare_field(field) + "#{@params[:fields_prefix]}#{field}" + end + + def default_params + { + :id => default_id, + :reset_strategy => default_reset_strategy + } + end + + def reset_source + end + + def reset_strategy + @params.fetch(:reset_strategy) + end + + private + + module Batch + def add_batch_to_processed_list(batch_id) + raise NotImplementedError + end + + def batch_already_processed?(batch_id) + raise NotImplementedError + end + + def batch_no_processed?(batch_id) + !batch_already_processed?(batch_id) + end + + def source_id + source.send(:source_id) + end + end + + include Batch + include CallbacksRb + end + end +end diff --git a/lib/treasury/storage/postgre_sql/base.rb b/lib/treasury/storage/postgre_sql/base.rb new file mode 100644 index 0000000..d01dd42 --- /dev/null +++ b/lib/treasury/storage/postgre_sql/base.rb @@ -0,0 +1,134 @@ +# coding: utf-8 + +module Treasury + module Storage + module PostgreSQL + # Хранилище на базе PosgreSQL. + # + # Может работать в 3-х режимах: + # - использовать "основное соединение" - то в котором работает сервис (там где хранится конфигурация); + # - использовать одно соединение с источником событий; + # - использовать свое соединение. + # + # По умолчанию, используется "основное соединение". + # Соединение можно указать через параметр :db_link_class. + # Указывается класс, исеющий метод connection (например модель ActiveRecord). + # + # В том случае, когда соединения хранилища и источника событий не совпадает (вариант 3) + # - в хранилище, ведется список обработанных батчей. + # + # Для этого в БД хранилища, должна существовать таблица: + # create table denormalization.processed_batches ( + # batch_id bigint not null, + # source_id character varying(32) not null, + # processed_at timestamp not null default NOW() + # ) + # + module Base + def start_transaction + return if transaction_started? + return unless storage_connection.outside_transaction? + internal_start_transaction + super + end + + def commit_transaction + return unless transaction_started? + internal_commit_transaction + super + end + + def rollback_transaction + return unless transaction_started? + internal_rollback_transaction + super + end + + # Public: Использует ли хранилище собственное соединение, + # не зависимое от соединения, в котором запрашиваются события. + # От этого зависит управление транзакцией записи данных в хранилище. + # + # Returns Boolean. + # + def use_own_connection? + processor_connection != storage_connection + end + + # Public: Возвращает соединение с БД, в котором запрашиваются события. + # + # Returns PostgreSQLAdapter. + # + def processor_connection + @processor_connection ||= source.send(:work_connection) + end + + # Public: Возвращает соединение с БД, для работы с хранилищем + # + # Returns PostgreSQLAdapter. + # + def storage_connection + @storage_connection ||= if params[:db_link_class].present? + params[:db_link_class].constantize.connection + else + source.send(:main_connection) + end + end + + alias_method :connection, :storage_connection + + protected + + def reset_source + @storage_connection = nil + @processor_connection = nil + end + + def internal_start_transaction + storage_connection.begin_db_transaction + storage_connection.increment_open_transactions + end + + def internal_commit_transaction + storage_connection.commit_db_transaction + storage_connection.decrement_open_transactions + end + + def internal_rollback_transaction + storage_connection.rollback_db_transaction + storage_connection.decrement_open_transactions + end + + def quote(str) + connection.quote(str) + end + + module Batch + def add_batch_to_processed_list(batch_id) + # для хранилищ, использующих сессию процессора, + # список обработанных батчей не ведется + return if use_processor_connection? + + connection.execute <<-SQL + INSERT INTO denormalization.processed_batches + VALUES + ( #{batch_id}, #{quote(source_id)} ) + SQL + end + + def batch_already_processed?(batch_id) + raise NotImplementedError if use_processor_connection? + + connection.select_value(<<-SQL).eql?('1') + SELECT 1 FROM denormalization.processed_batches + WHERE batch_id = #{batch_id} + AND source_id = #{quote(source_id)} + LIMIT 1 + SQL + end + end + + include Batch + end + end + end +end diff --git a/lib/treasury/storage/postgre_sql/db.rb b/lib/treasury/storage/postgre_sql/db.rb new file mode 100644 index 0000000..4b12861 --- /dev/null +++ b/lib/treasury/storage/postgre_sql/db.rb @@ -0,0 +1,222 @@ +# coding: utf-8 + +module Treasury + module Storage + module PostgreSQL + # Хранилище, пишущее данные в таблицу в БД. + # + # Данные пишутся по столбцам, по ключевому полю + # параметры: + # :table => schema.table - таблица хранилище + # :key => column - ключевое поле + # :db_link_class => model_class - класс соединения с БД (опционально) + # + # Поддерживается типизация полей. + # Для этого в настройках поля, укажите опцию :type, например: + # :fields => {:domain_level => {:type => :integer}} + # По умолчанию - все данные - строки. + # + class Db < Treasury::Storage::Base + include Treasury::Storage::PostgreSQL::Base + + DEFAULT_ID = :db + DEFAULT_SHARED = true + BULK_WRITE_THRESHOLD = 5 + + self.default_reset_strategy = :reset + + def transaction_bulk_write(data) + transaction { bulk_write(data) } + fire_callback(:after_transaction_bulk_write, self) + end + + def source_table(data_rows) + rows = data_rows.map do |object, row| + "(#{([quote(object)] + row.map { |_field, value| quote(value) }).join(',')})" + end + + fields = ([key] + data_rows.values.first.keys.map { |field| field }).join(',') + + <<-SQL.strip_heredoc + ( + VALUES + #{rows.join(',')} + ) t (#{fields}) + SQL + end + + def selected_fields_expression(data_rows) + (["#{key}#{key_cast_expression}"] + data_rows.values.first.keys.map { |field| "#{field}#{field_type(field)}" }).join(',') + end + + def matching_expression + "target.#{key} = source.#{key}" + end + + def updating_expression(data_rows) + if (first_row = data_rows.values.first).present? + first_row.keys.map { |field| "#{prepare_field(field)} = source.#{field}" }.join(',') + else + "#{key} = source.#{key}" + end + end + + def target_fields_expression(data_rows) + (["#{key}"] + data_rows.values.first.keys.map { |field| "#{prepare_field(field)}" }).join(',') + end + + def bulk_write(data_rows) + if data_rows.size <= BULK_WRITE_THRESHOLD + data_rows.each { |object, row| write(object, row) } + else + to_delete = data_rows.select { |object, row| row.nil? }.keys + delete_rows(to_delete) + data_rows.delete_if { |object, row| row.nil? } + return unless data_rows.present? + + ::PgTools::Merge.execute( + target_table: table_name, + source_table: source_table(data_rows), + selected_fields_expression: selected_fields_expression(data_rows), + matching_expression: matching_expression, + updating_expression: updating_expression(data_rows), + target_fields_expression: target_fields_expression(data_rows), + connection: connection + ) + end + end + + def read(object, field) + connection.select_value <<-SQL + SELECT "#{prepare_field(field)}" + FROM #{table_name} + WHERE "#{key}" = #{quote(object)}#{key_cast_expression} + SQL + end + + def reset_data(objects, fields) + case reset_strategy + when :delete + objects.nil? ? delete_all : delete_rows(objects) + when :reset + reset_all(fields) + else + raise NotImplementedError + end + end + + def delete_rows(objects) + objects = Array.wrap(objects).compact.uniq + return 0 unless objects.present? + + object_list = + case key_type + when :integer + objects.map(&:to_i).join(',') + else + objects.map { |object| quote(object.to_s) }.join(',') + end + + connection.update <<-SQL + DELETE FROM #{table_name} + WHERE #{key} IN (#{object_list}) + SQL + end + + def delete_all + connection.update <<-SQL + DELETE FROM #{table_name}; + SQL + end + + def reset_all(fields) + connection.execute <<-SQL + UPDATE #{table_name} + SET + #{fields.map { |f| "\"#{prepare_field(f)}\" = NULL" }.join(',')} + WHERE + "#{key}" IS NOT NULL + SQL + end + + # блокирует хранилище, если оно являются разделяемыми, т.е. те, + # в которые может вестись запись параллельно, несколькими обрабочиками разных полей. + # в этом случае может возникнуть следующая систуация: + # одно DB-хранилище и несколько полей привязанных к нему, + # при инициализации одного из полей в условиях большого потока обрабатываемых событий, + # может возникнуть ошибка: could not serialize access due to concurrent update + def lock + return unless shared? + start_transaction + connection.execute <<-SQL + LOCK TABLE #{table_name} IN SHARE MODE + SQL + end + + protected + + def default_id + DEFAULT_ID + end + + def table_name + connection.quote_table_name(params[:table]) + end + + def key + params[:key] + end + + def write(object, row) + return delete_rows(object) if row.nil? + + data_rows = {object => row} + ::PgTools::Merge.new( + :target_table => table_name, + :source_table => source_table(data_rows), + :selected_fields_expression => selected_fields_expression(data_rows), + :matching_expression => matching_expression, + :updating_expression => updating_expression(data_rows), + :target_fields_expression => target_fields_expression(data_rows), + :connection => connection + ).execute + end + + # Protected: Возвращает выражения для приведения типа поля, если тип поля задан. + # + # Returns String. + # + def field_type(field) + type = @params.fetch(:fields).try(:[], field).try(:[], :type) + "::#{type}" unless type.blank? + end + + # Protected: Возвращает выражения для приведения типа ключевого поля. + # + # Returns String. + # + def key_cast_expression + "::#{key_type}" + end + + # Protected: Возвращает тип ключевого поля. + # + # Returns Symbol. + # + def key_type + @key_type ||= @params.fetch(:key_type, :integer).to_sym + end + + private + + # признак является ли хранилище разделяемыми, т.е. таким, + # которое может использоваться разными полями одновременно + # значение данного признака устанавливается в параметрах конкретного хранилища, для конкретного поля: + # :params => {:shared => boolean value} + def shared? + @params[:shared] || DEFAULT_SHARED + end + end + end + end +end diff --git a/lib/treasury/storage/postgre_sql/pgq_producer.rb b/lib/treasury/storage/postgre_sql/pgq_producer.rb new file mode 100644 index 0000000..8def8d1 --- /dev/null +++ b/lib/treasury/storage/postgre_sql/pgq_producer.rb @@ -0,0 +1,70 @@ +# coding: utf-8 + +module Treasury + module Storage + module PostgreSQL + # Хранилище, пишущее данные в очередь Pgq. + # + # Добавляет в очередь с именем queue, события типа EVENT_TYPE. + # Данные передаются в виде url-encoded строки: "key=_processor_id_=processor_class&object&field=value&..." + # _processor_id_ - имя класса процессора, источника данных. + # + # параметры: + # :queue => queue_name - имя очереди + # :key => column - ключевое поле + # :db_link_class => model_class - класс соединения с БД (опционально) + # + # События удаления игнорируются. + # + class PgqProducer < Treasury::Storage::Base + include Treasury::Storage::PostgreSQL::Base + + DEFAULT_ID = :pgq + EVENT_TYPE = Pgq::Event::TYPE_INSERT + + def transaction_bulk_write(data) + transaction { bulk_write(data) } + fire_callback(:after_transaction_bulk_write, self) + end + + def bulk_write(data) + data.each { |object, row| write(object, row) } + end + + def reset_data(objects, fields) + end + + protected + + def default_id + DEFAULT_ID + end + + def queue + @queue ||= params[:queue] + end + + def event_type + @event_type ||= "#{EVENT_TYPE}:id" + end + + def key + params[:key] + end + + # Protected: Добавляет событие в очередь PGQ. + # + # Returns nothing. + # + def write(object, row) + return if row.nil? + + prepared_row = {key => object, :_processor_id_ => source.class.name} + prepared_row.merge!(row.inject(Hash.new) { |result, (field, value)| result.merge!(prepare_field(field) => value) }) + data = Rack::Utils.build_query(prepared_row) + ActiveRecord::Base.pgq_insert_event(queue, event_type, data) + end + end + end + end +end diff --git a/lib/treasury/storage/redis/base.rb b/lib/treasury/storage/redis/base.rb new file mode 100644 index 0000000..9c55918 --- /dev/null +++ b/lib/treasury/storage/redis/base.rb @@ -0,0 +1,145 @@ +# coding: utf-8 + +module Treasury + module Storage + module Redis + # базовый класс хранилища на основе Redis + # из-за особеностей реализации транзакционности в Redis, + # а именно невозможности чтения данных, пока выполняется транзакция, + # используется 2 соединения с Redis - для чтения и записи + class Base < Treasury::Storage::Base + RESET_FIELDS_BATCH_SIZE = 5000 + RESET_FIELDS_BATCH_PAUSE = Rails.env.staging? || !Rails.env.production? ? 0.seconds : 3.seconds + + self.default_reset_strategy = :delete + + def transaction_bulk_write(data) + write_session.pipelined do + start_transaction + data.each { |object, value| internal_write(object, value) } + fire_callback(:after_transaction_bulk_write, self) + commit_transaction + end + end + + def bulk_write(data) + write_session.pipelined do + data.each { |object, value| internal_write(object, value) } + end + end + + def reset_data(objects, fields) + reset_fields(fields) + end + + def start_transaction + return if transaction_started? + write_session.multi + super + end + + def commit_transaction + return unless transaction_started? + write_session.exec + super + end + + def rollback_transaction + return unless transaction_started? + write_session.discard + super + end + + def use_own_connection? + true + end + + protected + + def read_session + Treasury.configuration.redis + end + + def write_session + @@write_session ||= self.class.new_redis_session + end + + def hset(object, field, value) + write_session.hset(key(object), prepare_field(field), value) + end + + def hget(object, field) + read_session.hget(key(object), prepare_field(field)) + end + + def hdel(object, field) + write_session.hdel(key(object), prepare_field(field)) + end + + def delete(object) + write_session.del(key(object)) + end + + def expire(object, timeout) + write_session.expire(key(object), timeout) + end + + def reset_hash_fields(hash_key, fields) + fields = fields.map { |field| prepare_field(field) } + + read_session.keys(hash_key).in_groups_of(RESET_FIELDS_BATCH_SIZE, false) do |group| + write_session.pipelined do + group.each { |key| write_session.hdel(key, fields) } + end + sleep(RESET_FIELDS_BATCH_PAUSE) + end + end + + def object_key + "#{Treasury::ROOT_REDIS_KEY}:#{params[:key]}" + end + + def key(object) + "#{object_key}:#{object}" + end + + def reset_fields(fields) + reset_hash_fields("#{object_key}*", fields) + end + + def internal_write(object, data) + data.present? ? write(object, data) : delete(object) + end + + def self.new_redis_session + ::Redis.new(Treasury.configuration.redis.client.options) + end + + module Batch + PROCESSED_BATCHES_KEY = "#{Treasury::ROOT_REDIS_KEY}:processed_batches" + PROCESSED_BATCHES_EXPIRE_AFTER = 3.days + + def add_batch_to_processed_list(batch_id) + key = processed_batch_key(batch_id) + write_session.set(key, nil) + write_session.expire(key, PROCESSED_BATCHES_EXPIRE_AFTER) + end + + def batch_already_processed?(batch_id) + read_session.exists(processed_batch_key(batch_id)) + end + + protected + + def processed_batch_key(batch_id) + namespace = source_id + namespace = "#{namespace}:" if namespace.present? + "#{PROCESSED_BATCHES_KEY}:#{namespace}#{batch_id}" + end + end + + include Batch + end + end + end +end diff --git a/lib/treasury/storage/redis/multi.rb b/lib/treasury/storage/redis/multi.rb new file mode 100644 index 0000000..6e0f4d2 --- /dev/null +++ b/lib/treasury/storage/redis/multi.rb @@ -0,0 +1,28 @@ +# coding: utf-8 + +module Treasury + module Storage + module Redis + # хранилище на основе Redis + # данные хранятся в виде хэша строк + class Multi < Base + DEFAULT_ID = :redis_multi + + def read(object, field) + hget(object, field) + end + + protected + + def write(object, data) + data.each { |field, value| hset(object, field, value) } + expire(object, params[:expire]) if params[:expire] + end + + def default_id + DEFAULT_ID + end + end + end + end +end diff --git a/lib/treasury/supervisor.rb b/lib/treasury/supervisor.rb new file mode 100644 index 0000000..da5ff76 --- /dev/null +++ b/lib/treasury/supervisor.rb @@ -0,0 +1,136 @@ +# coding: utf-8 + +module Treasury + class Supervisor + STATE_RUNNING = 'running'.freeze + STATE_STOPPED = 'stopped'.freeze + + PROCESS_LOOP_SLEEP_TIME = 2.second + MAX_INITIALIZERS = 1 + + LOGGER_FILE_NAME = "#{ROOT_LOGGER_DIR}/supervisor".freeze + + module Errors + class SupervisorError < StandardError; end + end + + def self.run + supervisor = Models::SupervisorStatus.first + self.new(supervisor).process + end + + def process + logger.warn "Supervisor запущен" + begin + return unless check_active + set_state(STATE_RUNNING) + clear_last_error + while true + break unless check_terminate + run_initializers + run_workers + sleep(PROCESS_LOOP_SLEEP_TIME) + end + rescue => e + log_error(e) + raise + # TODO: нужно убрать райз и сделать свою отправку ошибки, будет надежнее + ensure + set_state(STATE_STOPPED) rescue nil + logger.warn "Supervisor остановлен" + end + end + + protected + + def initialize(supervisor_info) + @process_object = supervisor_info + end + + def run_initializers + in_initialize = Models::Field.in_initialize.select { |field| process_is_alive?(field.pid) }.size + available_for_run = [MAX_INITIALIZERS - in_initialize, 0].max + fields = fields_for_initialize.take(available_for_run) + #logger.debug "run_initializers, available_for_run = #{available_for_run}, fields.count = #{fields.count}" + fields.each { |field| run_initializer(field) } + end + + def fields_for_initialize + Models::Field + .active + .for_initialize_or_in_initialize + .ordered + .select do |field| + field.state.eql?(Fields::STATE_NEED_INITIALIZE) || + field.state.eql?(Fields::STATE_IN_INITIALIZE) && process_is_alive?(field.pid) + end + end + + def run_initializer(field) + @client = BgExecutor::Client.instance + @job_id = @client.queue_job!('treasury/initialize_field', field_class: field.field_class) + logger.info "Запущен джоб иницилизации поля #{quote(field.field_class)}, job_id = #{@job_id}" + rescue => e + logger.error "Ошибка при запуске джоба иницилизации поля #{quote(field.field_class)}:" + log_error(e) + end + + def run_workers + avaliable_workers.each { |worker| run_worker(worker) } + end + + def run_worker(worker) + return if process_is_alive?(worker.pid) + + @client = BgExecutor::Client.instance + @job_id = @client.queue_job!('treasury/worker', worker_id: worker.id) + + logger.warn "Запущен один воркер [#{worker.name}] для обработки всех полей!" if universal_worker? + logger.info "Запущен джоб воркер #{quote(worker.id)}, job_id = #{@job_id}" + rescue => e + logger.error "Ошибка при запуске воркера #{quote(worker.id)}:" + log_error(e) + end + + # Public: Возвращает массив воркеров, доступных для запуска. + # + # Returns Array of Denormalization::Models::Worker. + # + def avaliable_workers + workers = Models::Worker.active + + if universal_worker? + Array.wrap(workers.detect { |worker| worker.name.eql?('common') } || workers.first) + else + workers + end + end + + # Public: Обрабатывать ли все поля, в рамках одного воркера. + # + # В окружении, отличном от production, все поля обрабатываются одним воркером. + # + # Returns Boolean. + # + def universal_worker? + return @universal_worker unless @universal_worker.nil? + @universal_worker = !Rails.env.production? || Rails.env.staging? + end + + def check_active + return true if @process_object.active? + logger.warn "Supervisor не активен" + false + end + + include Treasury::Utils + include Treasury::Session + include Treasury::Logging + include Errors + + #self.logger_file_name :supervisor + def self.logger_default_file_name + LOGGER_FILE_NAME + end + end +end diff --git a/lib/treasury/utils.rb b/lib/treasury/utils.rb new file mode 100644 index 0000000..31c422c --- /dev/null +++ b/lib/treasury/utils.rb @@ -0,0 +1,100 @@ +# coding: utf-8 + +module Treasury + module Utils + extend ActiveSupport::Concern + + PROJECT_ROOT_PATH = Regexp.new(Rails.root.to_s) + OWN_GEMS_PATH = Regexp.new("apress|treasury") + + included do + attr_reader :process_object + end + + module ClassMethods + def connection + ActiveRecord::Base.connection + end + + def quote(value) + ActiveRecord::Base.connection.quote(value) + end + + def log_error(exception) + backtrace = exception.backtrace.select { |line| line =~ PROJECT_ROOT_PATH || line =~ OWN_GEMS_PATH } + error_message = "#{exception.message}\n\n #{backtrace.join("\n")}" + logger.error error_message + error_message + rescue + nil + end + + def current_method_name + if /`(.*)'/.match(caller.first) + return $1 + end + nil + end + end + + module InstanceMethods + def connection + self.class.connection + end + + def quote(value) + self.class.quote(value) + end + + def log_error(exception) + save_error(self.class.log_error(exception)) + rescue + end + + def check_terminate + refresh_state + return true unless @process_object.need_terminate? + logger.warn "Принят сигнал на завершение работы" + @process_object.need_terminate = false + @process_object.save! + false + end + + def check_state(state) + @process_object.state == state + end + + def set_state(state) + return set_session if check_state(state) && process_is_alive?(@process_object.pid) + @process_object.state = state + set_session + @process_object.save! + logger.info "Установлен статус %s" % [quote(@process_object.state)] + end + + def set_session + @process_object.pid = pid + @process_object.save! + logger.info "Установлен PID %s" % [quote(@process_object.pid)] + end + + def save_error(error_message) + error_message = error_message[1, 4000] unless error_message.nil? + @process_object.last_error = error_message + @process_object.save! + end + + def clear_last_error + save_error(nil) + end + + def current_method_name + self.class.current_method_name + end + + def refresh_state + @process_object.reload + end + end + end +end diff --git a/lib/treasury/worker.rb b/lib/treasury/worker.rb new file mode 100644 index 0000000..7ee1aa0 --- /dev/null +++ b/lib/treasury/worker.rb @@ -0,0 +1,174 @@ +# coding: utf-8 + +module Treasury + class Worker + STATE_RUNNING = 'running'.freeze + STATE_STOPPED = 'stopped'.freeze + + REFRESH_FIELDS_LIST_PERIOD = Rails.env.staging? || !Rails.env.production? ? 1.minute : 1.minute + IDLE_MAX_LAG = 5.minutes + PROCESS_LOOP_NORMAL_SLEEP_TIME = 0.05.seconds + PROCESS_LOOP_IDLE_SLEEP_TIME = 5.seconds + + LOGGER_FILE_NAME = "#{ROOT_LOGGER_DIR}/workers/%{name}_worker".freeze + + module Errors + class WorkerError < StandardError; end + class UnknownWorkerError < StandardError; end + end + + cattr_accessor :name + + def self.run(worker_id) + worker = Models::Worker.find(worker_id) + raise UnknownWorkerError if worker.nil? + self.name = worker.name + self.new(worker).process + end + + def initialize(worker_info) + @process_object = worker_info + end + + def current_worker + @process_object + end + + def process + logger.warn "Worker запущен" + begin + return unless check_active + set_state(STATE_RUNNING) + clear_last_error + while true + break unless check_terminate + idle(process_fields) + end + rescue Exception => e + log_error(e) + raise + ensure + set_state(STATE_STOPPED) rescue nil + logger.warn "Worker остановлен" + end + end + + def idle(mode) + if mode + sleep(PROCESS_LOOP_NORMAL_SLEEP_TIME) + else + sleep(PROCESS_LOOP_IDLE_SLEEP_TIME) + end + end + + def process_fields + refresh_fields_list + + idle = true + total_lag = 0 + @processing_fields.each do |field| + begin + begin + next if field.need_initialize? + + field.processors.each do |processor| + start_time = Time.now + result_hash = processor.processor_class.constantize.new(processor, logger).process + events_processed = result_hash[:events_processed] + unless events_processed.zero? + work_time = Time.now - start_time + + if work_time != 0 + logger.info "Обработано #{events_processed} событий, за #{work_time.round(1)}"\ + " с (#{(events_processed.to_f / work_time).round(1)} eps),"\ + " записано #{result_hash[:rows_written]} [#{processor.consumer_name}]" + end + end + + total_lag += @consumers_info[processor.consumer_name].try(:[], :seconds_lag).to_i + idle &&= events_processed.zero? + end + rescue Pgq::Errors::QueueOrSubscriberNotFoundError, Processors::Errors::InconsistencyDataError => e + # обработка исключений, требующих переиницилизации поля + logger.warn "Поле помечено как не иницилизированное." + field.need_initialize! + raise + end + rescue StandardError, NotImplementedError => e + logger.error "Ошибка при обработке поля #{field.title}:" + log_error(e) + end + end + + idle &&= total_lag < IDLE_MAX_LAG + !idle + end + + def refresh_fields_list + return if @last_update && (Time.now - @last_update < REFRESH_FIELDS_LIST_PERIOD) + + @processing_fields = processing_fields + + refresh_consumers_info + @last_update = Time.now + end + + # Public: Возвращает массив полей для обработки. + # + # В окружении, отличном от production, все поля обрабатываются одним воркером. + # + # Returns Array of Treasury::Models::Field. + # + def processing_fields + processing_fields = + Treasury::Models::Field + .for_processing + .joins(processors: :queue) + .eager_load(processors: :queue) + + if Rails.env.production? && !Rails.env.staging? + processing_fields = processing_fields.where(worker_id: current_worker.id) + end + + processing_fields.to_a + end + + def refresh_consumers_info + work_connections = + @processing_fields + .map(&:processors) + .flatten + .map(&:queue) + .map(&:work_connection) + .uniq + + @consumers_info = HashWithIndifferentAccess.new + work_connections.each do |connection| + Pgq::Consumer.get_consumer_info(connection).each do |consumer| + @consumers_info.merge!(consumer['consumer_name'] => HashWithIndifferentAccess.new(consumer)) + end + end + end + + def check_active + return true if @process_object.active? + logger.warn "Worker не активен" + false + end + + def main_connection + ActiveRecord::Base.connection + end + + include Treasury::Utils + include Treasury::Session + include Treasury::Logging + include Errors + + def self.logger_default_file_name + LOGGER_FILE_NAME % [name: name] + end + + self.logger_level = Logger::INFO + end +end diff --git a/spec/factories/field.rb b/spec/factories/field.rb new file mode 100644 index 0000000..c6d59a0 --- /dev/null +++ b/spec/factories/field.rb @@ -0,0 +1,36 @@ +# coding: utf-8 + +FactoryGirl.define do + factory 'denormalization/field', class: 'Treasury::Models::Field' do + sequence(:title) { |n| "title#{n}" } + sequence(:group) { |n| "group#{n}" } + field_class 'Treasury::Fields::Base' + active true + need_terminate false + state Treasury::Fields::STATE_INITIALIZED + sequence(:pid) { |n| n } + sequence(:progress) { |n| "progress_#{n}" } + snapshot_id '11:20:11,12,15' + last_error nil + worker_id nil + sequence(:oid) { |n| n } + params nil + storage [] + + trait :no_active do + active false + end + + trait :need_terminate do + need_terminate true + end + + trait :need_initialize do + state Treasury::Fields::STATE_NEED_INITIALIZE + end + + after(:create) do |field| + field.field_class.constantize._instance = nil + end + end +end diff --git a/spec/factories/processor.rb b/spec/factories/processor.rb new file mode 100644 index 0000000..664e3d1 --- /dev/null +++ b/spec/factories/processor.rb @@ -0,0 +1,10 @@ +# coding: utf-8 + +FactoryGirl.define do + factory 'denormalization/processor', class: 'Treasury::Models::Processor' do + association :queue, factory: 'denormalization/queue' + association :field, factory: 'denormalization/field' + processor_class 'Treasury::Processors::Base' + sequence(:consumer_name) { |n| "consumer_name#{n}" } + end +end diff --git a/spec/factories/queue.rb b/spec/factories/queue.rb new file mode 100644 index 0000000..8308a7e --- /dev/null +++ b/spec/factories/queue.rb @@ -0,0 +1,9 @@ +# coding: utf-8 + +FactoryGirl.define do + factory 'denormalization/queue', class: 'Treasury::Models::Queue' do + sequence(:name) { |n| "name#{n}" } + sequence(:table_name) { |n| "table_name#{n}" } + sequence(:trigger_code) { |n| "trigger_code#{n}" } + end +end diff --git a/spec/internal/config/database.yml b/spec/internal/config/database.yml new file mode 100644 index 0000000..c2fc41f --- /dev/null +++ b/spec/internal/config/database.yml @@ -0,0 +1,5 @@ +test: + adapter: postgresql + host: <%= ENV.fetch("TEST_DB_HOST", "localhost") %> + database: <%= ENV.fetch("TEST_DB_NAME", "docker") %> + username: <%= ENV.fetch("TEST_DB_USERNAME", "docker") %> diff --git a/spec/internal/config/initializers/treasury.rb b/spec/internal/config/initializers/treasury.rb new file mode 100644 index 0000000..b1ee51b --- /dev/null +++ b/spec/internal/config/initializers/treasury.rb @@ -0,0 +1,4 @@ +Treasury.configure do |config| + config.redis = MockRedis.new + config.job_error_notifications = ['test@test.com'] +end diff --git a/spec/internal/db/schema.rb b/spec/internal/db/schema.rb new file mode 100644 index 0000000..77ae0a2 --- /dev/null +++ b/spec/internal/db/schema.rb @@ -0,0 +1,2 @@ +ActiveRecord::Schema.define do +end diff --git a/spec/internal/log/test.log b/spec/internal/log/test.log deleted file mode 100644 index e69de29..0000000 diff --git a/spec/lib/treasury/fields/base_spec.rb b/spec/lib/treasury/fields/base_spec.rb new file mode 100644 index 0000000..9354de9 --- /dev/null +++ b/spec/lib/treasury/fields/base_spec.rb @@ -0,0 +1,40 @@ +# coding: utf-8 + +module FieldCallback + extend ActiveSupport::Concern + + included do + set_callback :data_changed, :after, :data_changed_callback + end + + module InstanceMethods + def data_changed_callback; end + end +end + +class TreasuryFieldsBase < Treasury::Fields::Base + include FieldCallback +end + +describe TreasuryFieldsBase do + subject { described_class.new(nil) } + + context '#data_changed' do + let(:chaged_objects) { [1, 2, 3] } + + it 'run callbacks' do + expect(subject).to receive(:run_callbacks).with(:data_changed, :after) + subject.send(:data_changed, chaged_objects) + end + + it 'correctly set changed_objects' do + subject.send(:data_changed, chaged_objects) + expect(subject.send(:changed_objects)).to eq chaged_objects + end + + it 'has data_changed callback' do + expect(subject).to receive(:data_changed_callback) + subject.send(:data_changed, chaged_objects) + end + end +end diff --git a/spec/lib/treasury/fields/delayed_spec.rb b/spec/lib/treasury/fields/delayed_spec.rb new file mode 100644 index 0000000..9b5fc1d --- /dev/null +++ b/spec/lib/treasury/fields/delayed_spec.rb @@ -0,0 +1,27 @@ +# coding: utf-8 + +class TreasuryFieldsBase < Treasury::Fields::Base + include Treasury::Fields::Delayed +end + +describe Treasury::Fields::Delayed do + Given(:field_model) { build_stubbed 'denormalization/field' } + Given(:field) { TreasuryFieldsBase.new(field_model) } + + context '#cancel_delayed_increments' do + after { field.send :cancel_delayed_increments } + + Given(:args) { ['field_class' => field_class] } + Given(:field_class) { double('field_class') } + + Then do + expect(Resque).to( + receive(:remove_delayed_selection). + with(Treasury::DelayedIncrementJob). + and_yield(args) + ) + end + + And { expect(field_class).to receive(:==).with(field_model.field_class) } + end +end diff --git a/spec/lib/treasury/fields/extractor_spec.rb b/spec/lib/treasury/fields/extractor_spec.rb index 899e33a..31c5741 100644 --- a/spec/lib/treasury/fields/extractor_spec.rb +++ b/spec/lib/treasury/fields/extractor_spec.rb @@ -1,6 +1,4 @@ -require 'spec_helper' - -RSpec.describe Treasury::Fields::Extractor do +describe Treasury::Fields::Extractor do let(:class_with_extractor) do Class.new do extend Treasury::Fields::Extractor diff --git a/spec/lib/treasury/fields/hash_operations_spec.rb b/spec/lib/treasury/fields/hash_operations_spec.rb index ad6c6ff..30c2c98 100644 --- a/spec/lib/treasury/fields/hash_operations_spec.rb +++ b/spec/lib/treasury/fields/hash_operations_spec.rb @@ -1,6 +1,4 @@ -require 'spec_helper' - -RSpec.describe Treasury::Fields::HashOperations do +describe Treasury::Fields::HashOperations do let(:hash_field_class) do Class.new { include Treasury::Fields::HashOperations } end @@ -20,4 +18,4 @@ expect(value_as_hash).to eq(10 => 100, 20 => 200) end end -end \ No newline at end of file +end diff --git a/spec/lib/treasury/processors/base_spec.rb b/spec/lib/treasury/processors/base_spec.rb index e59102a..10ca274 100644 --- a/spec/lib/treasury/processors/base_spec.rb +++ b/spec/lib/treasury/processors/base_spec.rb @@ -1,50 +1,578 @@ # coding: utf-8 -require 'spec_helper' +describe ::Treasury::Processors::Base do + # TODO: дописать тесты + # нужно дописать тесты методов: + # batch_already_processed? + # get_batch_events + # проверить что доступна @fetch_size и используется в get_batch_events + # process + # + # нужно написать интеграционный тест, которы йпроверит все цепочку от PGQ до Redis -RSpec.describe ::Treasury::Processors::Base do - let(:instance) { described_class.new } + class TestConsumer < Treasury::Processors::Base + def get_batch_events + @events_batches.each { |batch| yield batch } + end - let(:object) { '2000' } - let(:data) do - { - '1000' => {count1: '50', count2: '100'}, - '2000' => {count1: '550', count2: '600'} - } + def form_value(value) + {:field => value} + end end - let(:field) { double first_field: :count1 } - before do - instance.instance_variable_set(:@data, data) - instance.instance_variable_set(:@object, object) + BATCH_ID = 1 + # EVENTS_BATCHES_EMPTY = [] + EVENTS_BATCHES_TEST = [[1, 2, 3], [4], [2, 6]] + + let(:queue) { build 'denormalization/queue' } + let(:processor) { build 'denormalization/processor', queue: queue } + let(:consumer) { TestConsumer.new(processor) } + + subject { consumer } + + context 'метод write_data' do + it 'не должен ничего писать, если буфер пуст' do + data = [] + + storages = 2.times.map do + storage = Object.new + expect(storage).not_to receive(:bulk_write).with(data) + storage + end + + subject.instance_variable_set(:@data, data) + + allow(consumer).to receive(:storages).and_return(storages) + subject.send(:write_data) + end + + it 'должен для каждого хранилища выполнить методы start_transaction и bulk_write' do + data = [1, 2, 3] - allow(instance).to receive(:field).and_return(field) + storages = 2.times.map do + storage = Object.new + expect(storage).to receive(:bulk_write).with(data) + storage + end - allow(instance).to receive(:log_event) + subject.instance_variable_set(:@data, data) + + allow(consumer).to receive(:storages).and_return(storages) + subject.send(:write_data) + end end - describe '#current_value' do - it { expect(instance.current_value :count2).to eq '600' } + context 'метод commit_storage_transaction' do + it 'должен для каждого хранилища выполнить методы add_batch_to_processed_list и commit_transaction' do + batch_id = rand(100) + storages = 2.times.map do + storage = Object.new + expect(storage).to receive(:add_batch_to_processed_list).with(batch_id) + expect(storage).to receive(:commit_transaction) + storage + end + allow(consumer).to receive(:storages).and_return(storages) + + subject.instance_variable_set(:@batch_id, batch_id) + + subject.send(:commit_storage_transaction) + end end - describe '#object_value' do - before do - allow(instance.field).to receive(:raw_value).with('3000', :count2).and_return('1050') + context 'метод reset_buffer' do + it 'должен очищать буфер' do + subject.instance_variable_set(:@data, [1, 2, 3]) + subject.send(:reset_buffer) + expect(subject.instance_variable_get(:@data).size).to eq(0) + end + end + + context 'метод finish_batch' do + it 'должен вызывать метод API PGQ finish_batch с вернымыми параметрами' do + batch_id = rand(100) + subject.instance_variable_set(:@batch_id, batch_id) + expect(ActiveRecord::Base).to receive(:pgq_finish_batch).with(batch_id, subject.send(:work_connection)) + subject.send(:finish_batch) end + end - it do - expect(instance.object_value '1000', :count2).to eq '100' - expect(instance.object_value '1000').to eq '50' - expect(instance.object_value '2000').to eq '550' - expect(instance.object_value '3000', :count2).to eq '1050' + context 'метод interesting_event?' do + it 'должен опеределять интересные/не интересные события' do + event_txid = rand(100) + subject.event.txid = event_txid + enteresting = rand(2).to_s.to_b + snapshot = subject.instance_variable_get(:@snapshot) + expect(snapshot).to receive(:contains?).with(event_txid).and_return(enteresting) + expect(subject.send(:interesting_event?)).to be !enteresting end end - describe '#form_value' do - it { expect(instance.form_value '10').to eq '10' } + context 'метод get_batch' do + it 'должен вызывать метод API pgq_next_batch и устанавливать корректный @batch_id' do + batch_id = rand(100) + expect(ActiveRecord::Base).to( + receive(:pgq_next_batch) + .with(subject.queue_name, subject.consumer_name, subject.send(:work_connection)) + .and_return(batch_id) + ) + + subject.send(:get_batch) + expect(subject.instance_variable_get(:@batch_id)).to be batch_id + end + + it 'должен вызывать метод API pgq_next_batch и получать исключение Treasury::Pgq::Errors::QueueOrSubscriberNotFoundError, если не найдена очередь или консьюмер' do + allow(ActiveRecord::Base).to( + receive(:pgq_next_batch) + .and_raise(ActiveRecord::StatementInvalid.new('Not subscriber to queue')) + ) + + expect { subject.send(:get_batch) }.to raise_error(Treasury::Pgq::Errors::QueueOrSubscriberNotFoundError) + end + + it 'должен вызывать метод API pgq_next_batch и транслировать родительское исключение, если не обработано' do + allow(ActiveRecord::Base).to receive(:pgq_next_batch).and_raise(ActiveRecord::StatementInvalid) + expect { subject.send(:get_batch) }.to raise_error(ActiveRecord::StatementInvalid) + end end - describe '#result_row' do - it { expect(instance.result_row '10').to eq '2000' => '10' } + context 'метод process_events_batch' do + it 'должен вызывать метод обработки каждого события' do + events = rand(100).times.map { rand(100) } + events.each { |event| expect(subject).to receive(:internal_process_event).with(event).ordered } + subject.send(:process_events_batch, events) + end + end + + context 'метод internal_process_event' do + let(:event) { {rand(100) => rand(100)} } + + it 'должен ничего не делать и вернуть nil, если событие не интересно' do + subject.instance_variable_set(:@data, event) + allow(consumer).to receive(:interesting_event?).and_return(false) + allow(consumer.event).to receive(:assign) + expect(subject.send(:internal_process_event, {})).to be_nil + expect(subject.instance_variable_get(:@data)).to be event + end + + it 'не должен записывать в буфер данные, если результат обработки события nil' do + subject.instance_variable_set(:@data, event) + allow(consumer).to receive(:interesting_event?).and_return(true) + allow(consumer.event).to receive(:assign) + allow(subject).to receive(:process_event).and_return(nil) + expect(subject.instance_variable_get(:@data)).to be event + end + + it 'должен корректно обрабатывать событие' do + subject.instance_variable_set(:@data, {}) + expect(subject.event).to receive(:assign).with(event).ordered + expect(subject).to receive(:interesting_event?).ordered.and_return(true) + expect(subject).to receive(:init_event_params).ordered + expect(subject).to receive(:process_event).ordered.and_return(event) + + subject.send(:internal_process_event, event) + expect(subject.instance_variable_get(:@data)).to eq event + end + + context 'должен записывать все изменёные поля для нескольких событий' do + let(:event_1) { {'1' => {a: 1, b: 1}} } + let(:event_2) { {'1' => {b: 2}} } + + before do + subject.instance_variable_set(:@data, {}) + allow(subject).to receive(:interesting_event?).and_return(true) + allow(subject).to receive(:init_event_params) + allow(subject.event).to receive(:assign) + end + + it do + expect(subject).to receive(:process_event).and_return(event_1) + subject.send(:internal_process_event, event_1) + expect(subject).to receive(:process_event).and_return(event_2) + subject.send(:internal_process_event, event_2) + + expect(subject.instance_variable_get(:@data)).to eq('1' => {a: 1, b: 2}) + end + end + end + + context 'метод storages' do + it 'должен возвращать field.storages' do + storages = rand(100).times.map { rand(100) } + allow(subject).to receive_message_chain("field.storages").and_return(storages) + expect(subject.send(:storages)).to eq storages + end + end + + context 'метод field_class' do + it 'должен возвращать field.storages' do + expect(subject.send(:field_class)).to eq processor.field.field_class + end + end + + context 'метод field' do + it 'должен возвращать экземпляр processor.field.field_class' do + expect(subject.send(:field)).to be_an_instance_of processor.field.field_class.constantize + end + end + + context 'при создании класса должен быть проиницилизирован экземпляр @event' do + it { expect(subject.event).to be_an_instance_of Treasury::Pgq::Event } + end + + context 'при создании класса должен быть проиницилизирован экземпляр @event' do + it { expect(subject.instance_variable_get(:@snapshot)).to be_an_instance_of Treasury::Pgq::Snapshot } + end + + context 'интерфейс класса' do + it 'методы обработки событий должны генерировать исключения' do + expect { subject.send(:process_insert) }.to raise_error(NotImplementedError) + expect { subject.send(:process_update) }.to raise_error(NotImplementedError) + expect { subject.send(:process_delete) }.to raise_error(NotImplementedError) + end + + context 'метод process_event' do + it 'должен генерировать исключение, при обработке события, неизвестного типа' do + expect { subject.send(:process_event) }.to raise_error(Treasury::Processors::Errors::UnknownEventTypeError) + end + + it 'должен выполнить метод process_insert, при обработке события, с типом INSERT' do + expect(subject.event).to receive(:type).and_return(Treasury::Pgq::Event::TYPE_INSERT) + expect(subject).to receive(:process_insert) + expect(subject).not_to receive(:process_update) + expect(subject).not_to receive(:process_delete) + subject.send(:process_event) + end + + it 'должен выполнить метод process_update, при обработке события, с типом UPDATE, если данные изменились' do + expect(subject.event).to receive(:type).and_return(Treasury::Pgq::Event::TYPE_UPDATE) + expect(subject.event).to receive(:data_changed?).and_return(true) + expect(subject).not_to receive(:process_insert) + expect(subject).to receive(:process_update) + expect(subject).not_to receive(:process_delete) + subject.send(:process_event) + end + + it 'не должен выполнить метод process_update, при обработке события, с типом UPDATE, если данные не изменились' do + expect(subject.event).to receive(:type).and_return(Treasury::Pgq::Event::TYPE_UPDATE) + expect(subject).not_to receive(:process_insert) + expect(subject).not_to receive(:process_update) + expect(subject).not_to receive(:process_delete) + subject.send(:process_event) + end + + it 'должен выполнить метод process_update, при обработке события, с типом DELETE' do + expect(subject.event).to receive(:type).and_return(Treasury::Pgq::Event::TYPE_DELETE) + expect(subject).not_to receive(:process_insert) + expect(subject).not_to receive(:process_update) + expect(subject).to receive(:process_delete) + subject.send(:process_event) + end + end + + context 'метод init_params' do + it 'должен выполняться при создании объекта' do + expect_any_instance_of(TestConsumer).to receive(:init_params) + TestConsumer.new(processor) + end + + it 'должен заполнять параметры процессора' do + expect(subject.params).to eq processor.params || HashWithIndifferentAccess.new + expect(subject.instance_variable_get(:@params)).to eq processor.params || HashWithIndifferentAccess.new + end + end + + context 'метод nullify_current_value' do + it 'должен корректно работать' do + expect(subject.send(:nullify_current_value)).to eq subject.send(:result_row, nil) + end + end + + context '#delete_current_row' do + let(:delete_current_row) { consumer.send(:delete_current_row) } + + it { expect(delete_current_row).to eq(subject.object => nil) } + end + + context '#delete_current_value' do + let(:delete_current_value) { consumer.send(:delete_current_value) } + + context 'when processor is master' do + before { processor.params = {:master => true} } + + it { expect(delete_current_value).to eq(consumer.object => nil) } + end + + context 'when processor is not master' do + before { processor.params = {:master => false} } + + it { expect(delete_current_value).to eq consumer.send(:result_row, nil) } + end + end + + context 'метод no_action' do + it { expect(subject.send(:no_action)).to be_nil } + end + + context 'метод form_value' do + value = rand(100) + it { expect(subject.send(:form_value, value)).to eq(:field => value) } + end + + context 'метод result_row' do + it 'должен корректно работать' do + value = rand(100) + object = rand(100) + subject.instance_variable_set(:@object, object) + expect(subject.send(:result_row, value)).to eq(object => {:field => value}) + end + end + + context 'метод current_value' do + let(:field_value) { rand(100) } + let(:value) { {field: (field_value + 1), field2: (field_value + 2)} } + let(:object) { rand(100) } + + before do + subject.instance_variable_set(:@object, object) + subject.instance_variable_set(:@data, object => value) + end + + it 'должен читать значение из хранилища, если данных нет в буфере' do + subject.instance_variable_set(:@object, object + 1) + subject.send(:field).should_receive(:raw_value).and_return(field_value) + subject.send(:current_value, :field).should eq field_value + end + + it 'должен читать значение из хранилища, если поля нет в буфере' do + expect(subject.send(:field)).to receive(:raw_value).with(object, :field3).and_return(field_value) + expect(subject.send(:current_value, :field3)).to eq field_value + end + + it 'должен читать данные из буфера, если они там есть' do + expect(subject.send(:current_value, :field2)).to eq(value[:field2]) + end + + context 'если поле не указанно' do + it 'должен возвращать значение первого поля, если оно есть в буфере' do + expect(subject.send(:field)).to receive(:first_field).at_least(:once).and_return(:field) + expect(subject.send(:current_value)).to eq value[:field] + end + + it 'должен читать значение из хранилища, если поля нет в буфере' do + expect(subject.send(:field)).to receive(:first_field).at_least(:once).and_return(:field3) + expect(subject.send(:field)).to receive(:raw_value).with(object, nil).and_return(field_value) + expect(subject.send(:current_value)).to eq field_value + end + end + end + + context 'метод current_value_as_integer' do + it 'должен вызывать current_value с полем, если поле указано' do + field = :field + expect(subject).to receive(:current_value).with(field) + subject.send(:current_value_as_integer, field) + end + + it 'должен вызывать current_value с nil, если поле не указано' do + expect(subject).to receive(:current_value).with(nil) + subject.send(:current_value_as_integer) + end + + it 'должен возвращать правильный результат' do + value = rand(100) + allow(subject).to receive(:current_value).and_return(value.to_s) + expect(subject.send(:current_value_as_integer)).to eq value + end + end + + context 'метод incremented_current_value' do + it 'должен вызывать current_value_as_integer с полем, если поле указано' do + field = :field + expect(subject).to receive(:current_value_as_integer).with(field).and_return(0) + subject.send(:incremented_current_value, field) + end + + it 'должен вызывать current_value_as_integer с nil, если поле не указано' do + expect(subject).to receive(:current_value_as_integer).with(nil).and_return(0) + subject.send(:incremented_current_value) + end + + it 'должен возвращать правильный результат' do + value = rand(100) + allow(subject).to receive(:current_value_as_integer).and_return(value) + expect(subject.send(:incremented_current_value)).to eq value.succ + end + end + + context 'метод decremented_current_value' do + it 'должен вызывать current_value_as_integer с полем, если поле указано' do + field = :field + expect(subject).to receive(:current_value_as_integer).with(field).and_return(0) + subject.send(:decremented_current_value, field) + end + + it 'должен вызывать current_value_as_integer с nil, если поле не указано' do + expect(subject).to receive(:current_value_as_integer).with(nil).and_return(0) + subject.send(:decremented_current_value) + end + + it 'должен возвращать правильный результат' do + value = rand(100) + allow(subject).to receive(:current_value_as_integer).and_return(value) + expect(subject.send(:decremented_current_value)).to eq value.pred + end + end + + # перенести в processors::single spec + #context 'метод increment_current_value' do + # let(:value) { {:field => rand(100), :field2 => rand(100)} } + # let(:object) { rand(100) } + # + # before do + # subject.instance_variable_set(:@object, object) + # end + # + # it 'должен корректно работать при вызове с указанием поля' do + # subject.send(:increment_current_value, :field2).should eq subject.send(:result_row, value.merge(:field2 => value[:field2].next)) + # end + #end + end + + # FIXME: тесты ужасны, они проверяют реализацию, а не результат, любой рефакторинг окажется самым страшным адом. + describe '#process' do + let(:process) { subject.process } + + it 'должен получить следующий необработанный батч, если батча нет, ничего не делать' do + expect(ActiveRecord::Base).to receive(:pgq_next_batch).and_return(nil) + expect(subject).to_not receive(:start_storage_transaction) + expect(subject).not_to receive(:commit_storage_transaction) + expect(subject).not_to receive(:finish_batch) + expect(process).to eq(:events_processed => 0, :rows_written => 0) + end + + context 'если есть батч' do + context 'и батч не пустой' do + When do + allow(ActiveRecord::Base).to receive(:pgq_next_batch).and_return(BATCH_ID) + allow(subject).to receive(:process_events_batch) do |events| + data = subject.instance_variable_get(:@data) + subject.instance_variable_set(:@data, data.merge(events.inject({}) { |r, id| r.merge(id => 1) })) + end + subject.instance_variable_set(:@events_batches, EVENTS_BATCHES_TEST) + end + + context 'when called' do + Given(:storages) { [double('storage1'), double('storage2')] } + + before { allow(consumer).to receive(:storages).and_return(storages) } + + after { process } + + # порядок выполнения методов должен быть верным и колбеки должны вызываться в верном порядке + Then { expect(subject).to receive(:before_batch_processing).ordered } + + And { storages.each { |storage| expect(storage).to receive(:source=).with(subject) } } + + And { storages.each { |storage| expect(storage).to receive(:start_transaction) } } + + # для каждого батча, должен быть вызван метод обработки + And do + EVENTS_BATCHES_TEST.each { |batch| expect(subject).to receive(:process_events_batch).with(batch).ordered } + end + + And { expect(subject).to receive(:write_data).ordered } + + And { expect(subject).to receive(:after_batch_processing).ordered } + + # должен быть вызван метод подтверждения данных в хранилище, если записаны события + And { expect(subject).to receive(:commit_storage_transaction).ordered } + + # всегда должен вызывать метод завершения батча + And { expect(subject).to receive(:finish_batch).ordered } + + And { expect(subject).to receive(:data_changed).ordered } + end + + context 'after call' do + before do + allow(subject).to receive(:commit_storage_transaction) + allow(subject).to receive(:finish_batch) + process + end + + it 'метод должен вернуть корректное кол-во обработанных событий' do + expect(process).to eq(:events_processed => 6, :rows_written => 5) + end + + it 'должен верно формировать список измененных объектов' do + expect(subject.instance_variable_get(:@changed_keys)).to eq EVENTS_BATCHES_TEST.flatten.uniq + end + end + end + + context 'и батч не пустой', pending: 'unimplemented' do + it 'то для каждого батча должен быть вызван метод обработки' do + EVENTS_BATCHES_TEST.each { |batch| subject.should_receive(:process_events_batch).with(batch) } + subject.process + end + + it 'метод должен вернуть корректное кол-во обработанных событий' do + subject.process.should == {:events_processed => EVENTS_BATCHES_TEST.size, :rows_written => 0} + end + end + end + end + + context '#data_changed' do + before { subject.instance_variable_set(:@changed_keys, [1, 2, 3]) } + + context 'when changed objects list is empty' do + before { subject.instance_variable_set(:@changed_keys, []) } + + it do + expect(subject.send(:field)).not_to receive(:data_changed) + subject.send(:data_changed) + end + end + + context 'when changed objects list is not empty' do + it do + expect(subject.send(:field)).to receive(:data_changed).with([1, 2, 3]) + subject.send(:data_changed) + end + end + + context 'when exception in callbacks' do + before { allow(subject.send(:field)).to receive(:data_changed).and_raise(StandardError) } + it { expect { subject.send(:data_changed) }.to_not raise_error } + end + end + + describe '#object_value' do + let(:object) { '2000' } + let(:data) do + { + '1000' => {count1: '50', count2: '100'}, + '2000' => {count1: '550', count2: '600'} + } + end + let(:field) { double first_field: :count1 } + + before do + subject.instance_variable_set(:@data, data) + subject.instance_variable_set(:@object, object) + + allow(field).to receive(:raw_value).with('3000', :count2).and_return('1050') + allow(subject).to receive(:field).and_return(field) + + allow(subject).to receive(:log_event) + end + + it :aggregate_failures do + expect(subject.object_value('1000', :count2)).to eq '100' + expect(subject.object_value('1000')).to eq '50' + expect(subject.object_value('2000')).to eq '550' + expect(subject.object_value('3000', :count2)).to eq '1050' + end end -end \ No newline at end of file +end diff --git a/spec/lib/treasury/processors/counters_spec.rb b/spec/lib/treasury/processors/counters_spec.rb index 95daf7e..8119fba 100644 --- a/spec/lib/treasury/processors/counters_spec.rb +++ b/spec/lib/treasury/processors/counters_spec.rb @@ -1,7 +1,6 @@ -require 'spec_helper' - -RSpec.describe ::Treasury::Processors::Counters do - let(:processor) { processor_class.new } +describe ::Treasury::Processors::Counters do + let(:queue) { build 'denormalization/queue' } + let(:processor) { processor_class.new(build('denormalization/processor', queue: queue)) } let(:processor_class) do Class.new(::Treasury::Processors::Base) do @@ -71,7 +70,7 @@ def count?(data) context 'using raw data' do let(:processor_class) do - Class.new do + Class.new(::Treasury::Processors::Base) do include ::Treasury::Processors::Counters counters :count, fast_parsing: true diff --git a/spec/lib/treasury/processors/delayed_spec.rb b/spec/lib/treasury/processors/delayed_spec.rb new file mode 100644 index 0000000..3106714 --- /dev/null +++ b/spec/lib/treasury/processors/delayed_spec.rb @@ -0,0 +1,118 @@ +# coding: utf-8 + +class TestConsumer < Treasury::Processors::Base + include Treasury::Processors::Delayed +end + +describe Treasury::Processors::Delayed do + Given(:queue) { FactoryGirl.build 'denormalization/queue' } + Given(:processor) { FactoryGirl.build 'denormalization/processor', queue: queue } + Given(:consumer) { TestConsumer.new(processor) } + + When do + consumer.event.data[:id] = double('id') + consumer.object = double('object') + end + + describe '#delayed_increment_current_value' do + Given(:delayed_increment_current_value) { consumer.delayed_increment_current_value(:field_name, 48.hours) } + + context 'when call' do + after { delayed_increment_current_value } + + Then do + expect(Resque).to receive(:enqueue_in).with( + 48.hours, + Treasury::DelayedIncrementJob, + id: consumer.event.data[:id], + object: consumer.object, + field_class: consumer.send(:field_class), + field_name: :field_name, + by: 1 + ) + end + end + + Then { expect(delayed_increment_current_value).to eq consumer.send(:no_action) } + end + + describe '#cancel_delayed_increment' do + context 'when call' do + after { consumer.cancel_delayed_increment(:field_name) } + + Then do + expect(Resque).to receive(:remove_delayed).with( + Treasury::DelayedIncrementJob, + id: consumer.event.data[:id], + object: consumer.object, + field_class: consumer.send(:field_class), + field_name: :field_name, + by: 1 + ).and_return 0 + end + end + + context 'when removed job exists' do + When { allow(Resque).to receive(:remove_delayed).and_return 1 } + + Then { expect(consumer.cancel_delayed_increment(:field_name)).to be_truthy } + end + + context 'when removed job not exists' do + When { allow(Resque).to receive(:remove_delayed).and_return 0 } + + Then { expect(consumer.cancel_delayed_increment(:field_name)).to be_falsey } + end + end + + describe '#delayed_decrement_current_value' do + Given(:delayed_decrement_current_value) { consumer.delayed_decrement_current_value(:field_name, 48.hours) } + + context 'when call' do + after { delayed_decrement_current_value } + + Then do + expect(Resque).to receive(:enqueue_in).with( + 48.hours, + Treasury::DelayedIncrementJob, + id: consumer.event.data[:id], + object: consumer.object, + field_class: consumer.send(:field_class), + field_name: :field_name, + by: -1 + ) + end + end + + Then { expect(delayed_decrement_current_value).to eq consumer.send(:no_action) } + end + + describe '#cancel_delayed_decrement' do + context 'when call' do + after { consumer.cancel_delayed_decrement(:field_name) } + + Then do + expect(Resque).to receive(:remove_delayed).with( + Treasury::DelayedIncrementJob, + id: consumer.event.data[:id], + object: consumer.object, + field_class: consumer.send(:field_class), + field_name: :field_name, + by: -1 + ).and_return 0 + end + end + + context 'when removed job exists' do + When { allow(Resque).to receive(:remove_delayed).and_return 1 } + + Then { expect(consumer.cancel_delayed_decrement(:field_name)).to be_truthy } + end + + context 'when removed job not exists' do + When { allow(Resque).to receive(:remove_delayed).and_return 0 } + + Then { expect(consumer.cancel_delayed_decrement(:field_name)).to be_falsey } + end + end +end diff --git a/spec/lib/treasury/processors/hash_operations_spec.rb b/spec/lib/treasury/processors/hash_operations_spec.rb index 745cb16..b59dd69 100644 --- a/spec/lib/treasury/processors/hash_operations_spec.rb +++ b/spec/lib/treasury/processors/hash_operations_spec.rb @@ -1,5 +1,4 @@ # coding: utf-8 -require 'spec_helper' describe ::Treasury::Processors::HashOperations do let(:dummy_class) do diff --git a/spec/lib/treasury/storage/pgq_producer_spec.rb b/spec/lib/treasury/storage/pgq_producer_spec.rb new file mode 100644 index 0000000..f59b3cf --- /dev/null +++ b/spec/lib/treasury/storage/pgq_producer_spec.rb @@ -0,0 +1,49 @@ +# coding: utf-8 + +class ProcessorClass +end + +describe Treasury::Storage::PostgreSQL::PgqProducer do + let(:options) { {queue: 'queue', key: 'key'} } + let(:storage) { described_class.new(options) } + let(:event_type) { "#{Treasury::Pgq::Event::TYPE_INSERT}:id" } + let(:data) do + { + object1: {field1: :value1, field2: :value2}, + object2: {field11: :value11, field22: :value22} + } + end + + subject { storage } + + context do + before do + allow(storage).to receive(:start_transaction) + allow(storage).to receive(:storage_connection).and_return(ActiveRecord::Base) + allow(storage).to receive(:source).and_return(ProcessorClass.new) + allow(ActiveRecord::Base).to receive(:pgq_insert_event) + expect(ActiveRecord::Base).to( + receive(:pgq_insert_event).with( + 'queue', + event_type, + 'key=object1&_processor_id_=ProcessorClass&field1=value1&field2=value2' + ) + ) + expect(ActiveRecord::Base).to( + receive(:pgq_insert_event).with( + 'queue', + event_type, + 'key=object2&_processor_id_=ProcessorClass&field11=value11&field22=value22' + ) + ) + end + + it '#bulk_write' do + subject.bulk_write(data) + end + + it '#transaction_bulk_write' do + subject.transaction_bulk_write(data) + end + end +end diff --git a/spec/lib/treasury/storage/redis/base_spec.rb b/spec/lib/treasury/storage/redis/base_spec.rb new file mode 100644 index 0000000..4d000cf --- /dev/null +++ b/spec/lib/treasury/storage/redis/base_spec.rb @@ -0,0 +1,36 @@ +# coding: utf-8 + +describe Treasury::Storage::Redis::Base do + let(:options) { {:key => 'key'} } + let(:singleton_write_session) { described_class.new(options).send(:write_session) } + let(:storage) { described_class.new(options) } + let(:another_storage) { described_class.new(options) } + + before { allow_any_instance_of(described_class).to receive(:default_id) } + before { allow(described_class).to receive(:new_redis_session) { MockRedis.new } } + + describe 'is used for writing a single connection to Redis' do + subject { singleton_write_session } + + it { is_expected.to be storage.send(:write_session) } + it { is_expected.to be another_storage.send(:write_session) } + end + + describe '#rollback_transaction' do + let(:write_session) { storage.send(:write_session) } + + before do + storage.start_transaction + write_session.set('key', 'value') + end + + after { storage.rollback_transaction } + + it { expect(write_session.exists('key')).to be_truthy } + + it do + storage.rollback_transaction + expect(write_session.exists('key')).to be_falsey + end + end +end diff --git a/spec/models/field_spec.rb b/spec/models/field_spec.rb new file mode 100644 index 0000000..a19c9ce --- /dev/null +++ b/spec/models/field_spec.rb @@ -0,0 +1,25 @@ +# coding: utf-8 + +describe Treasury::Models::Field do + context 'when check db structure' do + it { is_expected.to have_db_column(:title).of_type(:string).with_options(limit: 128, null: false) } + it { is_expected.to have_db_column(:group).of_type(:string).with_options(limit: 128, null: false) } + it { is_expected.to have_db_column(:field_class).of_type(:string).with_options(limit: 128, null: false) } + it { is_expected.to have_db_column(:active).of_type(:boolean).with_options(default: false, null: false) } + it { is_expected.to have_db_column(:need_terminate).of_type(:boolean).with_options(default: false, null: false) } + it { is_expected.to have_db_column(:state).of_type(:string).with_options(limit: 128, null: false) } + it { is_expected.to have_db_column(:pid).of_type(:integer) } + it { is_expected.to have_db_column(:progress).of_type(:string).with_options(limit: 128) } + it { is_expected.to have_db_column(:snapshot_id).of_type(:string).with_options(limit: 4000) } + it { is_expected.to have_db_column(:last_error).of_type(:string).with_options(limit: 4000) } + it { is_expected.to have_db_column(:worker_id).of_type(:integer) } + it { is_expected.to have_db_column(:oid).of_type(:integer).with_options(null: false) } + it { is_expected.to have_db_column(:params).of_type(:string).with_options(limit: 4000) } + it { is_expected.to have_db_column(:storage).of_type(:string).with_options(limit: 4000) } + end + + context 'when check associations' do + it { is_expected.to have_many(:processors) } + it { is_expected.to belong_to(:worker) } + end +end diff --git a/spec/models/queue_spec.rb b/spec/models/queue_spec.rb new file mode 100644 index 0000000..1785c1e --- /dev/null +++ b/spec/models/queue_spec.rb @@ -0,0 +1,43 @@ +# coding: utf-8 + +describe Treasury::Models::Queue do + context 'when check db structure' do + it { is_expected.to have_db_column(:name).of_type(:string).with_options(limit: 128, null: false) } + it { is_expected.to have_db_column(:table_name).of_type(:string).with_options(limit: 256, null: true) } + it { is_expected.to have_db_column(:trigger_code).of_type(:string).with_options(limit: 2000, null: true) } + it { is_expected.to have_db_column(:db_link_class).of_type(:string).with_options(limit: 256) } + end + + context 'when check associations' do + it { is_expected.to have_many(:processors) } + end + + describe '.generate_trigger' do + let(:trigger_params) { {} } + subject { described_class.generate_trigger(trigger_params) } + + context 'without conditions' do + it { expect(subject).not_to include_text 'WHEN' } + end + + context 'with conditions' do + let(:trigger_params) { {conditions: 'one condition AND another condition'} } + it { expect(subject).to include_text 'WHEN (one condition AND another condition)' } + end + + context 'with specified columns' do + let(:trigger_params) { {of_columns: [:user_id, :state]} } + + it do + expect(subject.gsub(/\s/,'')).to eq(<<-SQL.gsub(/\s/,'')) + CREATE TRIGGER %{trigger_name} + AFTER insert OR update OR delete + OF user_id,state + ON %{table_name} + FOR EACH ROW + EXECUTE PROCEDURE pgq.logutriga(%{queue_name}, 'backup'); + SQL + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3805ba4..8ed7e32 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,11 +1,31 @@ require 'bundler/setup' +require 'treasury' + require 'simplecov' +require 'mock_redis' +require 'combustion' +require "factory_girl_rails" +require 'shoulda-matchers' + SimpleCov.start do - minimum_coverage 95 + minimum_coverage 70 end -require 'combustion' -Combustion.initialize! :action_mailer +Combustion.initialize! :action_mailer, :active_record -Treasury::SpecHelpers.stub_core_denormalization +require 'rspec/rails' +require 'rspec/given' +require 'apress-rspec' + +redis = Treasury.configuration.redis +Redis.current = redis +Resque.redis = redis + +RSpec.configure do |config| + config.use_transactional_fixtures = true + config.include FactoryGirl::Syntax::Methods + + config.filter_run_including focus: true + config.run_all_when_everything_filtered = true +end diff --git a/treasury.gemspec b/treasury.gemspec index 3eff242..dd502dd 100644 --- a/treasury.gemspec +++ b/treasury.gemspec @@ -19,9 +19,21 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'rails', '>= 3.1.12', '< 4.1' spec.add_runtime_dependency 'daemons', '>= 1.1.9' + spec.add_runtime_dependency 'class_logger', '>= 1.0.1' + spec.add_runtime_dependency 'callbacks_rb', '>= 0.0.1' + spec.add_runtime_dependency 'redis', '>= 3.2.1' + spec.add_runtime_dependency 'pg', '>= 0.16' + spec.add_runtime_dependency 'pg_tools', '>= 1.2.0' + spec.add_runtime_dependency 'string_tools', '>= 0.6.1' + spec.add_runtime_dependency 'resque-integration', '>= 1.9' - spec.add_development_dependency 'rspec', '>= 3.1' + spec.add_development_dependency 'rspec-rails' + spec.add_development_dependency 'factory_girl_rails' + spec.add_development_dependency 'apress-rspec' spec.add_development_dependency 'simplecov' spec.add_development_dependency 'combustion', '>= 0.5.3' spec.add_development_dependency 'appraisal' + spec.add_development_dependency 'mock_redis' + spec.add_development_dependency 'rspec-given' + spec.add_development_dependency 'shoulda-matchers' end