From 06c3784c8072cdac769e5ca96554a174b287e1ab Mon Sep 17 00:00:00 2001 From: Yaroslav Nelin Date: Sat, 14 Oct 2023 22:24:17 +0300 Subject: [PATCH] Add new tables and refactor old poorly code (#39) * Add new tables * Refactor --- .ruby-version | 2 +- Gemfile | 3 +- Gemfile.lock | 60 ++-- cropio_api_structure.sql | 274 +++++++++++++++++- lib/app.rb | 2 +- lib/data_importer.rb | 29 ++ lib/data_integrity_checker.rb | 58 ++++ lib/downloader.rb | 102 ++----- lib/models/basic_model.rb | 2 +- lib/models/growth_scale.rb | 8 + lib/models/growth_stage.rb | 8 + lib/models/growth_stage_group.rb | 8 + .../growth_stage_structure_mapping_item.rb | 8 + lib/models/growth_stages_prediction.rb | 8 + ...enance_plan_row_spare_part_mapping_item.rb | 2 +- lib/models/scout_report_point.rb | 8 + ...out_report_point_growth_stage_structure.rb | 8 + lib/models/scout_report_point_issue.rb | 8 + .../scout_report_point_issue_plant_part.rb | 8 + lib/models/scout_report_point_measurement.rb | 8 + lib/models/scouting_task.rb | 8 + lib/resource_fetcher.rb | 29 ++ 22 files changed, 526 insertions(+), 125 deletions(-) create mode 100644 lib/data_importer.rb create mode 100644 lib/data_integrity_checker.rb create mode 100644 lib/models/growth_scale.rb create mode 100644 lib/models/growth_stage.rb create mode 100644 lib/models/growth_stage_group.rb create mode 100644 lib/models/growth_stage_structure_mapping_item.rb create mode 100644 lib/models/growth_stages_prediction.rb create mode 100644 lib/models/scout_report_point.rb create mode 100644 lib/models/scout_report_point_growth_stage_structure.rb create mode 100644 lib/models/scout_report_point_issue.rb create mode 100644 lib/models/scout_report_point_issue_plant_part.rb create mode 100644 lib/models/scout_report_point_measurement.rb create mode 100644 lib/models/scouting_task.rb create mode 100644 lib/resource_fetcher.rb diff --git a/.ruby-version b/.ruby-version index 2bf1c1c..be94e6f 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.3.1 +3.2.2 diff --git a/Gemfile b/Gemfile index e254a37..ef2ed50 100644 --- a/Gemfile +++ b/Gemfile @@ -1,8 +1,9 @@ source 'https://rubygems.org' -ruby '2.3.1' +ruby '3.2.2' gem 'activerecord' +gem 'activerecord-import' gem 'activesupport' gem 'cropio-ruby', '~> 0.3' gem 'oj' diff --git a/Gemfile.lock b/Gemfile.lock index a086011..efc2e7a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,38 +1,39 @@ GEM remote: https://rubygems.org/ specs: - activemodel (5.2.6) - activesupport (= 5.2.6) - activerecord (5.2.6) - activemodel (= 5.2.6) - activesupport (= 5.2.6) - arel (>= 9.0) - activesupport (5.2.6) + activemodel (6.1.7.2) + activesupport (= 6.1.7.2) + activerecord (6.1.7.2) + activemodel (= 6.1.7.2) + activesupport (= 6.1.7.2) + activerecord-import (1.4.1) + activerecord (>= 4.2) + activesupport (6.1.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - arel (9.0.0) - byebug (11.0.1) - concurrent-ruby (1.1.9) - cropio-ruby (0.32) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + byebug (11.1.3) + concurrent-ruby (1.2.2) + cropio-ruby (0.33) json rest-client (~> 2.1.0rc1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) - i18n (1.8.11) + i18n (1.12.0) concurrent-ruby (~> 1.0) - json (2.6.1) + json (2.6.3) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) - minitest (5.14.4) + mime-types-data (3.2023.0218.1) + minitest (5.15.0) netrc (0.11.0) - oj (3.10.8) - pg (1.2.3) + oj (3.14.2) + pg (1.4.6) rake (13.0.6) redis (3.3.5) rest-client (2.1.0) @@ -40,18 +41,21 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - thread_safe (0.3.6) - tzinfo (1.2.9) - thread_safe (~> 0.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.2) + zeitwerk (2.6.7) PLATFORMS - ruby + arm64-darwin-22 + x86_64-darwin-20 + x86_64-linux DEPENDENCIES activerecord + activerecord-import activesupport byebug cropio-ruby (~> 0.3) @@ -61,7 +65,7 @@ DEPENDENCIES redis (~> 3.2) RUBY VERSION - ruby 2.3.1p112 + ruby 3.2.2p53 BUNDLED WITH - 2.2.16 + 2.3.26 diff --git a/cropio_api_structure.sql b/cropio_api_structure.sql index 1df7cdf..cf4c3bf 100644 --- a/cropio_api_structure.sql +++ b/cropio_api_structure.sql @@ -1,3 +1,14 @@ +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'postgis') + THEN + CREATE EXTENSION postgis; + RAISE NOTICE 'PostGIS extension created.'; + ELSE + RAISE NOTICE 'PostGIS extension already exists.'; + END IF; +END $$; + CREATE TABLE additional_objects ( id integer NOT NULL, field_group_id integer, @@ -77,7 +88,7 @@ CREATE TABLE agro_operations ( covered_area_by_track double precision, machine_work_area double precision, calculated_work_area_updated_at timestamp without time zone, - additional_info character varying(255), + additional_info character varying, season integer NOT NULL, planned_area double precision DEFAULT 0 NOT NULL, completed_area double precision DEFAULT 0 NOT NULL, @@ -85,7 +96,7 @@ CREATE TABLE agro_operations ( planned_water_rate double precision DEFAULT 0 NOT NULL, fact_water_rate double precision DEFAULT 0 NOT NULL, agri_work_id integer, - operation_number character varying(255), + operation_number character varying, covered_area_hourly json DEFAULT '{}'::json NOT NULL, planned_row_spacing double precision, planned_depth double precision, @@ -315,7 +326,14 @@ CREATE TABLE history_items ( active boolean DEFAULT true NOT NULL, crop_id integer, till_type character varying(255), - additional_info character varying + additional_info character varying, + harvested_weight double precision, + yield_density double precision, + expected_yield double precision, + grain_class character varying, + grain_humidity double precision, + grain_garbage_admixture double precision, + external_id character varying(255) ); ALTER TABLE history_items ADD PRIMARY KEY (id); @@ -557,13 +575,13 @@ ALTER TABLE user_role_assignments ADD PRIMARY KEY (id); CREATE TABLE user_role_permissions ( id integer NOT NULL, - user_role_id integer NOT NULL, - access_level character varying(255) NOT NULL, - access_for character varying(255) NOT NULL, - subject_type character varying(255) NOT NULL, + user_role_id integer, + access_level character varying(255), + access_for character varying(255), + subject_type character varying(255), subject_id integer, - created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + created_at timestamp without time zone, + updated_at timestamp without time zone ); ALTER TABLE user_role_permissions ADD PRIMARY KEY (id); @@ -576,7 +594,7 @@ CREATE TABLE weather_history_items ( temperature_max decimal, precipitation decimal, snow decimal, - field_group_id integer NOT NULL + field_group_id integer ); ALTER TABLE weather_history_items ADD PRIMARY KEY (id); @@ -1122,7 +1140,7 @@ CREATE TABLE fuel_hourly_data_items object_id integer NOT NULL, hour_start timestamp without time zone NOT NULL, fuel_consumption double precision, - fuel_drain double precision, + fuel_drain jsonb DEFAULT '[]'::jsonb, refuel double precision, data_source_parameter_id integer, external_id character varying, @@ -1244,4 +1262,236 @@ CREATE TABLE fuel_movements updated_at timestamp without time zone NOT NULL ); -ALTER TABLE fuel_movements ADD PRIMARY KEY (id); \ No newline at end of file +ALTER TABLE fuel_movements ADD PRIMARY KEY (id); + +CREATE TABLE growth_scales +( + id integer NOT NULL, + name character varying, + standard_name character varying, + localized_name character varying, + hidden boolean DEFAULT false, + external_id character varying, + additional_info character varying, + description text, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE growth_scales ADD PRIMARY KEY (id); + +CREATE TABLE growth_stages +( + id integer NOT NULL, + growth_scale_id integer, + code character varying, + name character varying, + localized_name character varying, + external_id character varying, + additional_info character varying, + description text, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE growth_stages ADD PRIMARY KEY (id); + +CREATE TABLE growth_stage_groups +( + id integer NOT NULL, + growth_scale_id integer, + name character varying, + standard_name character varying, + localized_name character varying, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE growth_stage_groups ADD PRIMARY KEY (id); + +CREATE TABLE growth_stages_predictions +( + id integer NOT NULL, + history_item_id integer, + growth_scale_id integer, + year integer, + prediction_data jsonb DEFAULT '{}'::jsonb, + fact_data jsonb DEFAULT '{}'::jsonb, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE growth_stages_predictions ADD PRIMARY KEY (id); + +CREATE TABLE scout_report_points +( + id integer NOT NULL, + field_scout_report_id integer, + growth_stage_id integer, + longitude double precision, + latitude double precision, + scouting_task_point_id integer, + additional_info character varying, + description text, + name character varying, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scout_report_points ADD PRIMARY KEY (id); + +CREATE TABLE scout_report_point_growth_stage_structures +( + id integer NOT NULL, + scout_report_point_id integer, + growth_stage_structure_mapping_items jsonb DEFAULT '{}'::jsonb, + additional_info character varying, + description text, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scout_report_point_growth_stage_structures ADD PRIMARY KEY (id); + +CREATE TABLE scout_report_point_issues +( + id integer NOT NULL, + scout_report_point_id integer, + plant_threat_id integer, + threat_level character varying, + threat_stage character varying, + amount double precision, + damaged_area double precision, + economic_damage_threshold_exceeded boolean DEFAULT false, + amount_per_100_plants double precision, + number_pests_in_trap integer, + affected_plants double precision, + herbicide_efficiency double precision, + additional_info character varying, + description text, + longitude double precision, + latitude double precision, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scout_report_point_issues ADD PRIMARY KEY (id); + +CREATE TABLE scout_report_point_issue_plant_parts +( + id integer NOT NULL, + scout_repscout_report_point_issue_id integer, + plant_part character varying, + progress double precision, + spread double precision, + additional_info character varying, + description text, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scout_report_point_issue_plant_parts ADD PRIMARY KEY (id); + +CREATE TABLE scout_report_point_measurements +( + id integer NOT NULL, + scout_report_point_id integer, + measurement_type character varying, + density_of_planting_linear_row_width double precision, + density_of_planting_linear_length_of_row double precision, + density_of_planting_linear_rows_count integer, + density_of_planting_linear_plants_in_rows integer, + rate_of_tillers_number_of_stems integer, + rate_of_tillers_number_of_plants integer, + rate_of_productive_tillers_number_of_heads integer, + rate_of_productive_tillers_number_of_plants integer, + yield_forecast_head_number double precision, + yield_forecast_kernels_per_head integer, + yield_forecast_thousand_grain_weight double precision, + density_of_planting_square double precision, + stems_density double precision, + productive_stems_density double precision, + plant_height double precision, + ground_cover double precision, + ears_count double precision, + plants_density_estimate double precision, + field_germination double precision, + harvesting_humidity double precision, + productivity double precision, + longitude double precision, + latitude double precision, + calculated_value double precision, + additional_info character varying, + description text, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scout_report_point_measurements ADD PRIMARY KEY (id); + +CREATE TABLE scouting_tasks +( + id integer NOT NULL, + field_id integer, + responsible_id integer, + author_id integer, + start_time timestamp without time zone NOT NULL, + end_time timestamp without time zone NOT NULL, + description text, + status character varying, + image_type character varying, + image_date date, + image_source_sign character varying, + scout_report_template_id integer, + season integer, + agro_operation_id integer, + automatic_scouting_task_id integer, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE scouting_tasks ADD PRIMARY KEY (id); + +CREATE TABLE growth_stage_structure_mapping_items +( + id integer NOT NULL, + growth_scale_name character varying, + growth_stage_group_name character varying, + growth_stage_name character varying, + growth_stage_id integer, + growth_stage_fraction integer, + external_id character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + +ALTER TABLE growth_stage_structure_mapping_items ADD PRIMARY KEY (id); + +-- ---------------------------------------------------- +-- +-- CREATING UNIQUE INDEX FOR ID COLUMNS IN EVERY TABLE +-- +-- ---------------------------------------------------- + +DO $$ +DECLARE + rec record; + index_name text; +BEGIN + FOR rec IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' and table_type = 'BASE TABLE' and table_name <> 'spatial_ref_sys') + LOOP + index_name := rec.table_name || '_id_unique_idx'; + + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = index_name) + THEN + EXECUTE format('CREATE UNIQUE INDEX %I ON public.%I (id);', index_name, rec.table_name); + END IF; + END LOOP; +END $$; diff --git a/lib/app.rb b/lib/app.rb index 14aa401..7803f20 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -6,7 +6,7 @@ module App ROOT = File.expand_path('../..', __FILE__) DB_CONFIG = YAML.load_file File.join ROOT, 'config', 'database.yml' - ALLOWED_PERIODS = %w[hour day week month year] + ALLOWED_PERIODS = %w[minute hour day week month year] DB_CONFIG['db'].update(DB_CONFIG['db']) do |_, v| if v.is_a?(String) && v.include?('ENV') diff --git a/lib/data_importer.rb b/lib/data_importer.rb new file mode 100644 index 0000000..7f858ad --- /dev/null +++ b/lib/data_importer.rb @@ -0,0 +1,29 @@ +class DataImporter + + attr_reader :model_class, :logger, :data + + def initialize(model_class, logger, data) + @model_class = model_class + @logger = logger + @data = data + end + + def import_data + needed_attributes = model_class.new.attributes + models_records_to_import = [] + + data.each_with_index do |rec, i| + selected_attr = rec.attributes.select { |k, _v| needed_attributes.include?(k) } + models_records_to_import << selected_attr + logger.print_on_same_line "Creating #{i}..." + end + + logger.print_on_same_line "Importing data to database..." + i = 1 + models_records_to_import.each_slice(1000) do |batch| + model_class.upsert_all(batch, unique_by: :id) + logger.print_on_same_line "batch #{i}" + i += 1 + end + end +end diff --git a/lib/data_integrity_checker.rb b/lib/data_integrity_checker.rb new file mode 100644 index 0000000..2cb7098 --- /dev/null +++ b/lib/data_integrity_checker.rb @@ -0,0 +1,58 @@ +class DataIntegrityChecker + MODELS_WITHOUT_CLEANING_IN_LOCAL_DB = %i[Version].freeze + + attr_accessor :processing_history + + def initialize(processed_model, processing_history, logger) + @processed_model = processed_model + @logger = logger + @processing_history = processing_history + end + + def check(klass) + remove_deleted_records_in_db(klass) + check_data_integrity(klass) + end + + private + + def remove_deleted_records_in_db(model_class) + return if MODELS_WITHOUT_CLEANING_IN_LOCAL_DB.include?(model_name.to_sym) + + db_ids = model_class.all.pluck(:id) + ids_to_remove = db_ids - cropio_ids + return if ids_to_remove.empty? + + model_class.where(id: ids_to_remove).delete_all + end + + def check_data_integrity(model_class) + return if processing_history + + db_ids = model_class.all.pluck(:id) + ids = cropio_ids - db_ids + return if ids.empty? + + @logger.print "#{model_name.ljust(45)} | "\ + "There is #{ids.count} records that absent in local DB. "\ + 'Trying to recreate them...' + + ids.each_with_index do |id, i| + rec = @processed_model.find(id) + next if rec.nil? + + model_class.create_or_update(rec.attributes) + @logger.print_on_same_line "ReSaving #{i}..." + end + end + + private + + def model_name + @processed_model.name.demodulize + end + + def cropio_ids + @cropio_ids ||= @processed_model.ids + end +end diff --git a/lib/downloader.rb b/lib/downloader.rb index af07112..b1b333b 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -1,90 +1,48 @@ -require 'active_support' -require 'active_support/core_ext' -require 'cropio' -require_relative '../lib/app' -require_relative '../lib/logger' +require_relative 'app' +require_relative 'logger' +require_relative 'resource_fetcher' +require_relative 'data_integrity_checker' +require_relative 'data_importer' + Dir.glob(App::ROOT + '/lib/models/*', &method(:require)) class Downloader include Cropio::Resources - attr_reader :logger, :processed_model, :cropio_ids, :from_time, :to_time, - :processing_history - - MODELS_WITHOUT_CLEANING_IN_LOCAL_DB = %i[Version].freeze - DISABLED_MODELS = %i[SatelliteImage Version].freeze + attr_accessor :processed_model, :from_time, :to_time, + :processing_history, :logger def download_all_data - resources.sort.each do |model| + ResourceFetcher.resources.sort.each do |model| begin @processed_model = Object.const_get(model) - reset_processing_status - - logger.print_on_same_line "Downloading #{model} from Cropio... "\ - "From: #{from_time} To: #{to_time}" - - data_from_api = @processed_model.changes(from_time.to_s, to_time.to_s) - logger.print "#{model_name.to_s.ljust(45)} | Size: "\ - "#{data_from_api.size.to_s.ljust(13)} | "\ - "From: #{from_time} | To: #{to_time}" - model_class = Object.const_get("Model::#{model}") + update_processing_status + data = ResourceFetcher.fetch_data_for_model(model, from_time, to_time, logger) ActiveRecord::Base.transaction do - data_from_api.each_with_index do |rec, i| - model_class.create_or_update(rec.attributes) - logger.print_on_same_line "Saving #{i}..." - end - - remove_deleted_records_in_db(model_class) - App::REDIS.set(model_name.to_s, to_time) - - check_data_integrity(model_class) + DataImporter.new(model_class, logger, data).import_data + DataIntegrityChecker.new(processed_model, processing_history, logger).check(model_class) + App::REDIS.set(model_name, to_time) end rescue Exception => e logger.print "Problem with model #{model.to_s}" logger.print e.message end end - end - def resources - resources = - Cropio::Resources - .constants - .select { |c| Cropio::Resources.const_get(c).is_a? Class } - - resources - DISABLED_MODELS + App::ADDITIONAL_MODELS + logger.print "All done..." end - def remove_deleted_records_in_db(model_class) - return if MODELS_WITHOUT_CLEANING_IN_LOCAL_DB.include?(model_name.to_sym) - - db_ids = model_class.all.pluck(:id) - ids_to_remove = db_ids - cropio_ids - return if ids_to_remove.empty? + def update_processing_status + @processing_history = true - model_class.where(id: ids_to_remove).delete_all + @from_time = start_time_for_downloading_data + @to_time = end_time_for_downloading_data end - def check_data_integrity(model_class) - return if processing_history - - db_ids = model_class.all.pluck(:id) - ids = cropio_ids - db_ids - return if ids.empty? - - logger.print "#{model_name.to_s.ljust(45)} | "\ - "There is #{ids.count} records that absent in local DB. "\ - 'Trying to recreate them...' - - ids.each_with_index do |id, i| - rec = @processed_model.find(id) - next if rec.nil? - - model_class.create_or_update(rec.attributes) - logger.print_on_same_line "ReSaving #{i}..." - end + def start_time_for_downloading_data + App::REDIS.get(model_name) || App::START_DOWNLOAD_YEAR end def end_time_for_downloading_data @@ -97,24 +55,8 @@ def end_time_for_downloading_data supposed_time end - def reset_processing_status - @processing_history = true - @cropio_ids = nil - - @from_time = start_time_for_downloading_data - @to_time = end_time_for_downloading_data - end - - def start_time_for_downloading_data - App::REDIS.get(model_name) || App::START_DOWNLOAD_YEAR - end - def model_name - @processed_model.name.demodulize - end - - def cropio_ids - @cropio_ids ||= @processed_model.ids + processed_model.name.demodulize end def logger diff --git a/lib/models/basic_model.rb b/lib/models/basic_model.rb index 38f2511..9b3f39c 100644 --- a/lib/models/basic_model.rb +++ b/lib/models/basic_model.rb @@ -3,6 +3,6 @@ def create_or_update(hash) hash.select! {|k, _v| self.column_names.include? k } obj = self.find_or_initialize_by(id: hash['id']) obj.attributes = hash - obj.save + obj.save(:validate => false) end end diff --git a/lib/models/growth_scale.rb b/lib/models/growth_scale.rb new file mode 100644 index 0000000..b4ccf43 --- /dev/null +++ b/lib/models/growth_scale.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class GrowthScale < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/growth_stage.rb b/lib/models/growth_stage.rb new file mode 100644 index 0000000..2b55155 --- /dev/null +++ b/lib/models/growth_stage.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class GrowthStage < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/growth_stage_group.rb b/lib/models/growth_stage_group.rb new file mode 100644 index 0000000..1329224 --- /dev/null +++ b/lib/models/growth_stage_group.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class GrowthStageGroup < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/growth_stage_structure_mapping_item.rb b/lib/models/growth_stage_structure_mapping_item.rb new file mode 100644 index 0000000..38b3508 --- /dev/null +++ b/lib/models/growth_stage_structure_mapping_item.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class GrowthStageStructureMappingItem < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/growth_stages_prediction.rb b/lib/models/growth_stages_prediction.rb new file mode 100644 index 0000000..764c5e4 --- /dev/null +++ b/lib/models/growth_stages_prediction.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class GrowthStagesPrediction < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/maintenance_plan_row_spare_part_mapping_item.rb b/lib/models/maintenance_plan_row_spare_part_mapping_item.rb index 0d40e20..a743419 100644 --- a/lib/models/maintenance_plan_row_spare_part_mapping_item.rb +++ b/lib/models/maintenance_plan_row_spare_part_mapping_item.rb @@ -2,7 +2,7 @@ require_relative 'basic_model' module Model - class MaintenanceRecordRowSparePartMappingItem < ActiveRecord::Base + class MaintenancePlanRowSparePartMappingItem < ActiveRecord::Base extend BasicModel end end diff --git a/lib/models/scout_report_point.rb b/lib/models/scout_report_point.rb new file mode 100644 index 0000000..f76ff8a --- /dev/null +++ b/lib/models/scout_report_point.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutReportPoint < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/scout_report_point_growth_stage_structure.rb b/lib/models/scout_report_point_growth_stage_structure.rb new file mode 100644 index 0000000..09f3017 --- /dev/null +++ b/lib/models/scout_report_point_growth_stage_structure.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutReportPointGrowthStageStructure < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/scout_report_point_issue.rb b/lib/models/scout_report_point_issue.rb new file mode 100644 index 0000000..109b221 --- /dev/null +++ b/lib/models/scout_report_point_issue.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutReportPointIssue < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/scout_report_point_issue_plant_part.rb b/lib/models/scout_report_point_issue_plant_part.rb new file mode 100644 index 0000000..41e5b61 --- /dev/null +++ b/lib/models/scout_report_point_issue_plant_part.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutReportPointIssuePlantPart < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/scout_report_point_measurement.rb b/lib/models/scout_report_point_measurement.rb new file mode 100644 index 0000000..e956a0a --- /dev/null +++ b/lib/models/scout_report_point_measurement.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutReportPointMeasurement < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/models/scouting_task.rb b/lib/models/scouting_task.rb new file mode 100644 index 0000000..6e16e82 --- /dev/null +++ b/lib/models/scouting_task.rb @@ -0,0 +1,8 @@ +require 'active_record' +require_relative 'basic_model' + +module Model + class ScoutingTask < ActiveRecord::Base + extend BasicModel + end +end diff --git a/lib/resource_fetcher.rb b/lib/resource_fetcher.rb new file mode 100644 index 0000000..b548b55 --- /dev/null +++ b/lib/resource_fetcher.rb @@ -0,0 +1,29 @@ +require "cropio" + +module ResourceFetcher + include Cropio::Resources + + DISABLED_MODELS = %i[Version].freeze + + def self.fetch_data_for_model(model, from_time, to_time, logger) + processed_model = Object.const_get(model) + + logger.print_on_same_line "Downloading #{model} from Cropio... "\ + "From: #{from_time} To: #{to_time}" + + data = processed_model.changes(from_time.to_s, to_time.to_s) + logger.print "#{model.to_s.ljust(45)} | Size: "\ + "#{data.size.to_s.ljust(13)} | "\ + "From: #{from_time} | To: #{to_time}" + data + end + + def self.resources + @resources ||= + Cropio::Resources + .constants + .select { |c| Cropio::Resources.const_get(c).is_a? Class } + + @resources - DISABLED_MODELS + App::ADDITIONAL_MODELS + end +end