adds deduplicator to avoid duplicate urls scrape

deduper/deduper.go

@@ -0,0 +1,17 @@
+package deduper
+
+import (
+	"context"
+	"sync"
+)
+
+type Deduper interface {
+	AddIfNotExists(context.Context, string) bool
+}
+
+func New() Deduper {
+	return &hashmap{
+		seen: make(map[uint64]struct{}),
+		mux:  &sync.RWMutex{},
+	}
+}

deduper/hashmap.go

@@ -0,0 +1,42 @@
+package deduper
+
+import (
+	"context"
+	"hash/fnv"
+	"sync"
+)
+
+var _ Deduper = (*hashmap)(nil)
+
+type hashmap struct {
+	mux  *sync.RWMutex
+	seen map[uint64]struct{}
+}
+
+func (d *hashmap) AddIfNotExists(_ context.Context, key string) bool {
+	d.mux.RLock()
+	if _, ok := d.seen[d.hash(key)]; ok {
+		d.mux.RUnlock()
+		return false
+	}
+
+	d.mux.RUnlock()
+
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	if _, ok := d.seen[d.hash(key)]; ok {
+		return false
+	}
+
+	d.seen[d.hash(key)] = struct{}{}
+
+	return true
+}
+
+func (d *hashmap) hash(key string) uint64 {
+	h := fnv.New64()
+	h.Write([]byte(key))
+
+	return h.Sum64()
+}

gmaps/job.go

@@ -9,19 +9,31 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/google/uuid"
+	"github.com/gosom/google-maps-scraper/deduper"
 	"github.com/gosom/scrapemate"
 	"github.com/playwright-community/playwright-go"
 )
 
+type GmapJobOptions func(*GmapJob)
+
 type GmapJob struct {
 	scrapemate.Job
 
 	MaxDepth     int
 	LangCode     string
 	ExtractEmail bool
+
+	Deduper deduper.Deduper
 }
 
-func NewGmapJob(id, langCode, query string, maxDepth int, extractEmail bool, geoCoordinates string, zoom int) *GmapJob {
+func NewGmapJob(
+	id, langCode, query string,
+	maxDepth int,
+	extractEmail bool,
+	geoCoordinates string,
+	zoom int,
+	opts ...GmapJobOptions,
+) *GmapJob {
 	query = url.QueryEscape(query)
 
 	const (
@@ -55,9 +67,19 @@ func NewGmapJob(id, langCode, query string, maxDepth int, extractEmail bool, geo
 		ExtractEmail: extractEmail,
 	}
 
+	for _, opt := range opts {
+		opt(&job)
+	}
+
 	return &job
 }
 
+func WithDeduper(d deduper.Deduper) GmapJobOptions {
+	return func(j *GmapJob) {
+		j.Deduper = d
+	}
+}
+
 func (j *GmapJob) UseInResults() bool {
 	return false
 }
@@ -84,7 +106,10 @@ func (j *GmapJob) Process(ctx context.Context, resp *scrapemate.Response) (any,
 		doc.Find(`div[role=feed] div[jsaction]>a`).Each(func(_ int, s *goquery.Selection) {
 			if href := s.AttrOr("href", ""); href != "" {
 				nextJob := NewPlaceJob(j.ID, j.LangCode, href, j.ExtractEmail)
-				next = append(next, nextJob)
+
+				if j.Deduper == nil || j.Deduper.AddIfNotExists(ctx, href) {
+					next = append(next, nextJob)
+				}
 			}
 		})
 	}

runner/databaserunner/databaserunner.go

@@ -138,6 +138,7 @@ func (d *dbrunner) produceSeedJobs(ctx context.Context) error {
 		d.cfg.Email,
 		d.cfg.GeoCoordinates,
 		d.cfg.Zoom,
+		nil,
 	)
 	if err != nil {
 		return err

runner/filerunner/filerunner.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gosom/google-maps-scraper/deduper"
 	"github.com/gosom/google-maps-scraper/runner"
 	"github.com/gosom/google-maps-scraper/tlmt"
 	"github.com/gosom/scrapemate"
@@ -70,13 +71,16 @@ func (r *fileRunner) Run(ctx context.Context) (err error) {
 		_ = runner.Telemetry().Send(ctx, evt)
 	}()
 
+	dedup := deduper.New()
+
 	seedJobs, err = runner.CreateSeedJobs(
 		r.cfg.LangCode,
 		r.input,
 		r.cfg.MaxDepth,
 		r.cfg.Email,
 		r.cfg.GeoCoordinates,
 		r.cfg.Zoom,
+		dedup,
 	)
 	if err != nil {
 		return err

runner/jobs.go

@@ -9,11 +9,20 @@ import (
 	"plugin"
 	"strings"
 
+	"github.com/gosom/google-maps-scraper/deduper"
 	"github.com/gosom/google-maps-scraper/gmaps"
 	"github.com/gosom/scrapemate"
 )
 
-func CreateSeedJobs(langCode string, r io.Reader, maxDepth int, email bool, geoCoordinates string, zoom int) (jobs []scrapemate.IJob, err error) {
+func CreateSeedJobs(
+	langCode string,
+	r io.Reader,
+	maxDepth int,
+	email bool,
+	geoCoordinates string,
+	zoom int,
+	dedup deduper.Deduper,
+) (jobs []scrapemate.IJob, err error) {
 	scanner := bufio.NewScanner(r)
 
 	for scanner.Scan() {
@@ -29,7 +38,14 @@ func CreateSeedJobs(langCode string, r io.Reader, maxDepth int, email bool, geoC
 			id = strings.TrimSpace(after)
 		}
 
-		jobs = append(jobs, gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom))
+		opts := []gmaps.GmapJobOptions{}
+		if dedup != nil {
+			opts = append(opts, gmaps.WithDeduper(dedup))
+		}
+
+		job := gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom, opts...)
+
+		jobs = append(jobs, job)
 	}
 
 	return jobs, scanner.Err()

runner/webrunner/webrunner.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gosom/google-maps-scraper/deduper"
 	"github.com/gosom/google-maps-scraper/runner"
 	"github.com/gosom/google-maps-scraper/tlmt"
 	"github.com/gosom/google-maps-scraper/web"
@@ -172,13 +173,16 @@ func (w *webrunner) scrapeJob(ctx context.Context, job *web.Job) error {
 		coords = job.Data.Lat + "," + job.Data.Lon
 	}
 
+	dedup := deduper.New()
+
 	seedJobs, err := runner.CreateSeedJobs(
 		job.Data.Lang,
 		strings.NewReader(strings.Join(job.Data.Keywords, "\n")),
 		job.Data.Depth,
 		job.Data.Email,
 		coords,
 		job.Data.Zoom,
+		dedup,
 	)
 	if err != nil {
 		err2 := w.svc.Update(ctx, job)