Queue Depth functionality + Panic with Async

queue/queue.go

@@ -90,6 +90,7 @@ func (q *Queue) AddURL(URL string) error {
 	r := &colly.Request{
 		URL:    u2,
 		Method: "GET",
+		Depth:  1,
 	}
 	d, err := r.Marshal()
 	if err != nil {
@@ -131,6 +132,10 @@ func (q *Queue) Size() (int, error) {
 // to perform requests. Run blocks while the queue has active requests
 // The given Storage must not be used directly while Run blocks.
 func (q *Queue) Run(c *colly.Collector) error {
+	if c.Async {
+		// Async causes Queue crawling to silently fail, since all requests finish "Instantly"
+		panic("Cannot run Async collector in Queue!")
+	}
 	q.mut.Lock()
 	if q.wake != nil && q.running == true {
 		q.mut.Unlock()

queue/queue_test.go

@@ -1,6 +1,7 @@
 package queue
 
 import (
+	"fmt"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
@@ -75,6 +76,59 @@ func TestQueue(t *testing.T) {
 	}
 }
 
+func TestCollectorDepth(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(serverHandler))
+	defer server.Close()
+	maxDepth := 4
+	c := colly.NewCollector(
+		colly.MaxDepth(maxDepth),
+		colly.AllowURLRevisit(),
+	)
+
+	storage := &InMemoryQueueStorage{MaxSize: 100000}
+	q, err := New(10, storage)
+	if err != nil {
+		panic(err)
+	}
+
+	c.OnRequest(func(req *colly.Request) {
+		if req.Depth > maxDepth {
+			errMsg := fmt.Sprintf("Invalid depth value! Expected %d, got %d", maxDepth, req.Depth)
+			panic(errMsg)
+		}
+		req.Visit(server.URL)
+	})
+
+	q.AddURL(server.URL)
+
+	err = q.Run(c)
+	if err != nil {
+		t.Fatalf("Queue.Run() return an error: %v", err)
+	}
+}
+
+func TestAsyncPanic(t *testing.T) {
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("The code did not panic")
+		}
+	}()
+
+	c := colly.NewCollector(
+		colly.MaxDepth(5),
+		colly.AllowURLRevisit(),
+		colly.Async(true),
+	)
+
+	storage := &InMemoryQueueStorage{MaxSize: 100000}
+	q, err := New(10, storage)
+	if err != nil {
+		panic(err)
+	}
+
+	q.Run(c)
+}
+
 func serverHandler(w http.ResponseWriter, req *http.Request) {
 	if !serverRoute(w, req) {
 		shutdown(w)

request.go

@@ -158,7 +158,7 @@ func (r *Request) Retry() error {
 
 // Do submits the request
 func (r *Request) Do() error {
-	return r.collector.scrape(r.URL.String(), r.Method, r.Depth, r.Body, r.Ctx, *r.Headers, !r.collector.AllowURLRevisit)
+	return r.collector.scrape(r.URL.String(), r.Method, r.Depth+1, r.Body, r.Ctx, *r.Headers, !r.collector.AllowURLRevisit)
 }
 
 // Marshal serializes the Request