Merge pull request #11 from alphasights/ap-need_moar_time

Increase timeout allowed
alphasights · Dec 5, 2016 · b810a2d · b810a2d
2 parents 986031e + bdceb80
commit b810a2d
Showing 1 changed file with 8 additions and 3 deletions.
diff --git a/lib/crawl/engine.rb b/lib/crawl/engine.rb
@@ -62,9 +62,14 @@ def retrieve(page)
     absolute_url = options[:domain] + page.relative_url
 
     http = EventMachine::HttpRequest.new(absolute_url)
-    req = http.get :redirects => MAX_REDIRECTS, :head => {'authorization' => [options[:username], options[:password]]}
-    req.timeout(15)
-
+    req = http.get :redirects => MAX_REDIRECTS,
+                   :connect_timeout => 20,
+                   :inactivity_timeout => 20,
+                   :head => {
+                     'authorization' => [
+                       options[:username], options[:password]
+                      ]
+                    }
     req.errback do
       if req.nil?
         page.intermittent("Req is nil. WAT?")