From 431115fad3ec0569b9e65c05974765e0082e340f Mon Sep 17 00:00:00 2001
From: Sid Shanker <sshanker220@gmail.com>
Date: Sat, 19 Nov 2016 02:42:52 -0800
Subject: [PATCH] Updated load_drains rake task.

Made the following changes:

* Delete all drains from DB that are no longer in the
dataset.

* Update properties on existing drains
---
 app/models/thing.rb       | 37 +++++++++++++++++++++++++++++++++++++
 lib/tasks/data.rake       | 31 +------------------------------
 test/fixtures/things.yml  |  7 +++++++
 test/models/thing_test.rb | 22 ++++++++++++++++++++++
 4 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/app/models/thing.rb b/app/models/thing.rb
index e5591747..a446036c 100644
--- a/app/models/thing.rb
+++ b/app/models/thing.rb
@@ -1,3 +1,6 @@
+require 'open-uri'
+require 'csv'
+
 class Thing < ActiveRecord::Base
   extend Forwardable
   include ActiveModel::ForbiddenAttributesProtection
@@ -38,4 +41,38 @@ def detail_link
 
     nil
   end
+
+  def self.load_drains(source_url)
+    puts 'Downloading Drains... ... ...'
+    csv_string = open(source_url).read
+    drains = CSV.parse(csv_string, headers: true)
+    puts "Downloaded #{drains.size} Drains."
+
+    city_ids = drains.map { |drain|
+      drain["PUC_Maximo_Asset_ID"].gsub("N-", "")
+    }
+
+    Thing.where.not(city_id: city_ids).delete_all
+
+    drains.each do |drain|
+      next unless ['Storm Water Inlet Drain', 'Catch Basin Drain'].include?(drain['Drain_Type'])
+
+      (lat, lng) = drain['Location'].delete('()').split(',').map(&:strip)
+
+      thing_hash = {
+        name: drain['Drain_Type'],
+        system_use_code: drain['System_Use_Code'],
+        lat: lat,
+        lng: lng,
+      }
+
+      thing = Thing.where(city_id: drain['PUC_Maximo_Asset_ID'].gsub('N-', '')).first_or_initialize
+      if thing.new_record?
+        puts "Creating thing #{thing_hash[:city_id]}"
+      else
+        puts "Updating thing #{thing_hash[:city_id]}"
+      end
+      thing.update_attributes!(thing_hash)
+    end
+  end
 end
diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake
index d70c7b6e..5c3199c5 100644
--- a/lib/tasks/data.rake
+++ b/lib/tasks/data.rake
@@ -1,36 +1,7 @@
 require 'rake'
 
 namespace :data do
-  require 'open-uri'
-  require 'csv'
-
   task load_drains: :environment do
-    puts 'Downloading Drains... ... ...'
-    url = 'https://data.sfgov.org/api/views/jtgq-b7c5/rows.csv?accessType=DOWNLOAD'
-    csv_string = open(url).read
-    drains = CSV.parse(csv_string, headers: true)
-    puts "Downloaded #{drains.size} Drains."
-
-    drains.each do |drain|
-      next unless ['Storm Water Inlet Drain', 'Catch Basin Drain'].include?(drain['Drain_Type'])
-
-      (lat, lng) = drain['Location'].delete('()').split(',').map(&:strip)
-
-      thing_hash = {
-        name: drain['Drain_Type'],
-        system_use_code: drain['System_Use_Code'],
-        lat: lat,
-        lng: lng,
-      }
-
-      thing = Thing.where(city_id: drain['PUC_Maximo_Asset_ID'].gsub!('N-', '')).first_or_initialize
-      if thing.new_record?
-        puts "Updating thing #{thing_hash[:city_id]}"
-      else
-        puts "Creating thing #{thing_hash[:city_id]}"
-      end
-
-      thing.update_attributes!(thing_hash)
-    end
+    Thing.load_drains('https://data.sfgov.org/api/views/jtgq-b7c5/rows.csv?accessType=DOWNLOAD')
   end
 end
diff --git a/test/fixtures/things.yml b/test/fixtures/things.yml
index d72daf6c..29ba87eb 100644
--- a/test/fixtures/things.yml
+++ b/test/fixtures/things.yml
@@ -51,3 +51,10 @@ thing_10:
   city_id: 10
   lat: 42.371378
   lng: -71.038005
+
+thing_11:
+  city_id: 11
+  lat: 42.383339
+  lng: -71.049226
+  name: "Catch Basin Drain"
+  system_use_code: 'ABC'
diff --git a/test/models/thing_test.rb b/test/models/thing_test.rb
index 220c1c8b..72a7a644 100644
--- a/test/models/thing_test.rb
+++ b/test/models/thing_test.rb
@@ -15,4 +15,26 @@ class ThingTest < ActiveSupport::TestCase
     t.system_use_code = 'MS4'
     assert_equal 'http://sfwater.org/index.aspx?page=399', t.detail_link
   end
+
+  test 'loading drains, deletes existing drains not in data set, updates properties on rest' do
+    thing_1 = things(:thing_1)
+    thing_11 = things(:thing_11)
+
+    fake_url = 'http://sf-drain-data.org'
+    fake_response = [
+      "PUC_Maximo_Asset_ID,Drain_Type,System_Use_Code,Location",
+      "N-11,Catch Basin Drain,ABC,\"(37.75, -122.40)\"",
+    ].join("\n")
+    stub_request(:get, fake_url).to_return(body: fake_response)
+
+    Thing.load_drains(fake_url)
+    thing_11.reload
+
+    # Asserts thing_1 is deleted
+    assert_nil Thing.find_by(id: thing_1.id)
+
+    # Asserts properties on thing_11 have been updated
+    assert_equal thing_11.lat, BigDecimal.new(37.75, 16)
+    assert_equal thing_11.lng, BigDecimal.new(-122.40, 16)
+  end
 end