Updated load_drains rake task.

Gemfile

@@ -13,6 +13,7 @@ gem 'rails_admin'
 gem 'validates_formatting_of'
 
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
+gem 'paranoia', '~> 2.2'
 
 group :assets do
   gem 'sass-rails', '>= 4.0.3'

Gemfile.lock

@@ -110,6 +110,8 @@ GEM
       pkg-config (~> 1.1.7)
     obscenity (1.0.2)
     orm_adapter (0.5.0)
+    paranoia (2.2.0)
+      activerecord (>= 4.0, < 5.1)
     parser (2.3.0.7)
       ast (~> 2.2)
     pg (0.18.4)
@@ -236,6 +238,7 @@ DEPENDENCIES
   haml
   http_accept_language
   obscenity (~> 1.0, >= 1.0.2)
+  paranoia (~> 2.2)
   pg
   puma
   rails (~> 4.2.4)
@@ -255,4 +258,4 @@ RUBY VERSION
    ruby 2.2.3p173
 
 BUNDLED WITH
-   1.13.1
+   1.13.6

app/mailers/thing_mailer.rb

@@ -23,4 +23,12 @@ def reminder(thing)
     @user = thing.user
     mail(to: @user.email, subject: ['Remember to clear your adopted drain'])
   end
+
+  def drain_update_report(deleted_drains_with_adoptee, deleted_drains_no_adoptee, created_drains)
+    @deleted_drain_ids_with_adoptee = deleted_drains_with_adoptee.map(&:city_id)
+    @deleted_drain_ids_with_no_adoptee = deleted_drains_no_adoptee.map(&:city_id)
+    @created_drain_ids = created_drains.map(&:city_id)
+    subject = "Adopt-a-Drain import (#{deleted_drains_with_adoptee.count} adopted drains removed, #{created_drains.count} drains added, #{deleted_drains_no_adoptee.count} removed)"
+    mail(to: User.where(admin: true).pluck(:email), subject: subject)
+  end
 end

app/models/thing.rb

@@ -1,6 +1,13 @@
+require 'open-uri'
+require 'csv'
+
 class Thing < ActiveRecord::Base
+  acts_as_paranoid
   extend Forwardable
   include ActiveModel::ForbiddenAttributesProtection
+
+  VALID_DRAIN_TYPES = ['Storm Water Inlet Drain', 'Catch Basin Drain'].freeze
+
   belongs_to :user
   def_delegators :reverse_geocode, :city, :country, :country_code,
                  :full_address, :state, :street_address, :street_name,
@@ -15,6 +22,7 @@ def self.find_closest(lat, lng, limit = 10)
     query = <<-SQL
       SELECT *, (3959 * ACOS(COS(RADIANS(?)) * COS(RADIANS(lat)) * COS(RADIANS(lng) - RADIANS(?)) + SIN(RADIANS(?)) * SIN(RADIANS(lat)))) AS distance
       FROM things
+      WHERE deleted_at is NULL
       ORDER BY distance
       LIMIT ?
       SQL
@@ -38,4 +46,84 @@ def detail_link
 
     nil
   end
+
+  def self._get_parsed_drains_csv(source_url)
+    csv_string = open(source_url).read
+    CSV.parse(csv_string, headers: true)
+  end
+
+  def self._delete_non_existing_drains(drains_from_source)
+    city_ids = drains_from_source.map do |drain|
+      drain['PUC_Maximo_Asset_ID'].gsub('N-', '').to_i
+    end
+
+    things_to_delete = Thing.where.not(city_id: city_ids)
+    things_to_delete.each(&:destroy!)
+
+    things_to_delete.partition { |drain| drain.user_id.present? }
+  end
+
+  def self._drain_params(drain_blob)
+    (lat, lng) = drain_blob['Location'].delete('()').split(',').map(&:strip)
+    {
+      name: drain_blob['Drain_Type'],
+      system_use_code: drain_blob['System_Use_Code'],
+      lat: lat,
+      lng: lng,
+    }
+  end
+
+  def self.city_id_for_drain_blob(drain_blob)
+    drain_blob['PUC_Maximo_Asset_ID'].gsub('N-', '').to_i
+  end
+
+  def self.stored_city_ids
+    Thing.unscoped.pluck(:city_id)
+  end
+
+  def self._drain_blobs_to_update(drain_blobs)
+    drain_blobs.select do |drain_blob|
+      stored_city_ids.include?(city_id_for_drain_blob(drain_blob))
+    end
+  end
+
+  def self._update_drains(drain_blobs)
+    _drain_blobs_to_update(drain_blobs).each do |drain_blob|
+      thing = Thing.unscoped.find_by(city_id: city_id_for_drain_blob(drain_blob))
+      Rails.logger.info("Updating thing #{thing.city_id}")
+      thing.restore! if thing.deleted_at?
+      thing.update!(_drain_params(drain_blob))
+    end
+  end
+
+  def self._drain_blobs_to_create(drain_blobs)
+    drain_blobs.reject do |drain_blob|
+      (stored_city_ids.include?(city_id_for_drain_blob(drain_blob)) ||
+       !VALID_DRAIN_TYPES.include?(drain_blob['Drain_Type']))
+    end
+  end
+
+  def self._create_drains(drain_blobs)
+    _drain_blobs_to_create(drain_blobs).map do |drain_blob|
+      city_id = city_id_for_drain_blob(drain_blob)
+      thing = Thing.unscoped.find_or_initialize_by(city_id: city_id)
+      Rails.logger.info("Creating thing #{thing.city_id}")
+
+      thing.update!(_drain_params(drain_blob))
+      thing
+    end
+  end
+
+  def self.load_drains(source_url)
+    Rails.logger.info('Downloading Drains... ... ...')
+    drain_blobs = _get_parsed_drains_csv(source_url)
+    Rails.logger.info("Downloaded #{drain_blobs.size} Drains.")
+
+    deleted_drains_with_adoptee, deleted_drains_no_adoptee = _delete_non_existing_drains(drain_blobs)
+
+    _update_drains(drain_blobs)
+    created_drains = _create_drains(drain_blobs)
+
+    ThingMailer.drain_update_report(deleted_drains_with_adoptee, deleted_drains_no_adoptee, created_drains).deliver_now
+  end
 end

app/views/thing_mailer/drain_deleted_notification.html.erb

@@ -0,0 +1,3 @@
+Hi Adopt a drain admin!
+
+Just notifying you that drain with PUC_Maximo_Asset_ID: <%= @thing.city_id %>, and rails id <% @thing.id %>, has been removed from the database.

app/views/thing_mailer/drain_update_report.html.erb

@@ -0,0 +1,52 @@
+<h3>Drain Import Report</h3>
+
+<% if @deleted_drain_ids_with_adoptee.present? %>
+  <table border="1">
+    <thead>
+      <tr>
+        <td><b>Drains With Adoptee Removed</b></td>
+      </tr>
+    </thead>
+    <tbody>
+      <% @deleted_drain_ids_with_adoptee.each do |drain_id| %>
+        <tr>
+          <td><%= drain_id %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>
+
+<% if @deleted_drain_ids_with_no_adoptee.present? %>
+  <table border="1">
+    <thead>
+      <tr>
+        <td><b>Drains With No Adoptee Removed</b></td>
+      </tr>
+    </thead>
+    <tbody>
+      <% @deleted_drain_ids_with_no_adoptee.each do |drain_id| %>
+        <tr>
+          <td><%= drain_id %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>
+
+<% if @created_drain_ids.present? %>
+  <table border="1">
+    <thead>
+      <tr>
+        <td><b>Drains Added</b></td>
+      </tr>
+    </thead>
+    <tbody>
+      <% @created_drain_ids.each do |drain_id| %>
+        <tr>
+          <td><%= drain_id %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

db/migrate/20161205030306_add_deleted_at_to_things.rb

@@ -0,0 +1,6 @@
+class AddDeletedAtToThings < ActiveRecord::Migration
+  def change
+    add_column :things, :deleted_at, :datetime
+    add_index :things, :deleted_at
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160326200455) do
+ActiveRecord::Schema.define(version: 20161205030306) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -52,9 +52,11 @@
     t.integer  "city_id"
     t.integer  "user_id"
     t.string   "system_use_code"
+    t.datetime "deleted_at"
   end
 
   add_index "things", ["city_id"], name: "index_things_on_city_id", unique: true, using: :btree
+  add_index "things", ["deleted_at"], name: "index_things_on_deleted_at", using: :btree
 
   create_table "users", force: :cascade do |t|
     t.datetime "created_at"

lib/tasks/data.rake

@@ -1,36 +1,7 @@
 require 'rake'
 
 namespace :data do
-  require 'open-uri'
-  require 'csv'
-
   task load_drains: :environment do
-    puts 'Downloading Drains... ... ...'
-    url = 'https://data.sfgov.org/api/views/jtgq-b7c5/rows.csv?accessType=DOWNLOAD'
-    csv_string = open(url).read
-    drains = CSV.parse(csv_string, headers: true)
-    puts "Downloaded #{drains.size} Drains."
-
-    drains.each do |drain|
-      next unless ['Storm Water Inlet Drain', 'Catch Basin Drain'].include?(drain['Drain_Type'])
-
-      (lat, lng) = drain['Location'].delete('()').split(',').map(&:strip)
-
-      thing_hash = {
-        name: drain['Drain_Type'],
-        system_use_code: drain['System_Use_Code'],
-        lat: lat,
-        lng: lng,
-      }
-
-      thing = Thing.where(city_id: drain['PUC_Maximo_Asset_ID'].gsub!('N-', '')).first_or_initialize
-      if thing.new_record?
-        puts "Updating thing #{thing_hash[:city_id]}"
-      else
-        puts "Creating thing #{thing_hash[:city_id]}"
-      end
-
-      thing.update_attributes!(thing_hash)
-    end
+    Thing.load_drains('https://data.sfgov.org/api/views/jtgq-b7c5/rows.csv?accessType=DOWNLOAD')
   end
 end

test/fixtures/things.yml

@@ -51,3 +51,10 @@ thing_10:
   city_id: 10
   lat: 42.371378
   lng: -71.038005
+
+thing_11:
+  city_id: 11
+  lat: 42.383339
+  lng: -71.049226
+  name: "Catch Basin Drain"
+  system_use_code: 'ABC'

test/mailers/thing_mailer_test.rb

@@ -39,4 +39,21 @@ class ThingMailerTest < ActionMailer::TestCase
     assert_equal ['[email protected]'], email.to
     assert_equal 'We really do love you, Erik!', email.subject
   end
+
+  test 'drain_update_report' do
+    admin_1 = users(:admin)
+    admin_2 = users(:admin)
+    admin_2.update(email: '[email protected]')
+    email = nil
+    deleted_thing = things(:thing_1)
+
+    assert_emails(1) do
+      email = ThingMailer.drain_update_report([deleted_thing], [], []).deliver_now
+    end
+
+    assert_includes email.to, admin_1.email
+    assert_includes email.to, admin_2.email
+
+    assert_equal email.subject, 'Adopt-a-Drain import (1 adopted drains removed, 0 drains added, 0 removed)'
+  end
 end

test/models/thing_test.rb

@@ -15,4 +15,45 @@ class ThingTest < ActiveSupport::TestCase
     t.system_use_code = 'MS4'
     assert_equal 'http://sfwater.org/index.aspx?page=399', t.detail_link
   end
+
+  test 'loading drains, deletes existing drains not in data set, updates properties on rest' do
+    admin = users(:admin)
+    thing_1 = things(:thing_1)
+    thing_11 = things(:thing_11)
+
+    deleted_drain = things(:thing_3)
+    deleted_drain.destroy!
+
+    fake_url = 'http://sf-drain-data.org'
+    fake_response = [
+      'PUC_Maximo_Asset_ID,Drain_Type,System_Use_Code,Location',
+      'N-3,Catch Basin Drain,ABC,"(42.38, -71.07)"',
+      'N-11,Catch Basin Drain,ABC,"(37.75, -122.40)"',
+      'N-12,Catch Basin Drain,DEF,"(39.75, -121.40)"',
+    ].join("\n")
+    stub_request(:get, fake_url).to_return(body: fake_response)
+
+    Thing.load_drains(fake_url)
+
+    email = ActionMailer::Base.deliveries.last
+    assert_equal email.to, [admin.email]
+    assert_equal email.subject, 'Adopt-a-Drain import (0 adopted drains removed, 1 drains added, 9 removed)'
+    thing_11.reload
+
+    # Asserts thing_1 is deleted
+    assert_nil Thing.find_by(id: thing_1.id)
+
+    # Asserts thing_3 is reified
+    assert_equal Thing.find_by(city_id: 3).id, deleted_drain.id
+
+    # Asserts creates new drain
+    new_drain = Thing.find_by(city_id: 12)
+    assert_not_nil new_drain
+    assert_equal new_drain.lat, BigDecimal.new(39.75, 16)
+    assert_equal new_drain.lng, BigDecimal.new(-121.40, 16)
+
+    # Asserts properties on thing_11 have been updated
+    assert_equal thing_11.lat, BigDecimal.new(37.75, 16)
+    assert_equal thing_11.lng, BigDecimal.new(-122.40, 16)
+  end
 end