From a999b11b1c50db0a6ae3fb778df7c675bde0c863 Mon Sep 17 00:00:00 2001
From: Oliver Tari <oliver.tari@supercharge.io>
Date: Tue, 9 Jul 2024 14:59:03 +0200
Subject: [PATCH] respect robots.txt by default

---
 colly.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/colly.go b/colly.go
index ae74b7c3..f0c321e5 100644
--- a/colly.go
+++ b/colly.go
@@ -481,7 +481,7 @@ func (c *Collector) Init() {
 	c.wg = &sync.WaitGroup{}
 	c.lock = &sync.RWMutex{}
 	c.robotsMap = make(map[string]*robotstxt.RobotsData)
-	c.IgnoreRobotsTxt = true
+	c.IgnoreRobotsTxt = false
 	c.ID = atomic.AddUint32(&collectorCounter, 1)
 	c.TraceHTTP = false
 	c.Context = context.Background()
@@ -1468,7 +1468,6 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader {
 	}
 	buffer.WriteString(dashBoundary + "--\n\n")
 	return bytes.NewReader(buffer.Bytes())
-
 }
 
 // randomBoundary was borrowed from