Skip to content

Commit

Permalink
Fix exit due to inactivity
Browse files Browse the repository at this point in the history
  • Loading branch information
gosom committed Oct 19, 2024
1 parent 584fbaa commit e176084
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 6 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
APP_NAME := google_maps_scraper
VERSION := 1.4.1
VERSION := 1.4.2

default: help

Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ mkdir -p gmapsdata && docker run -v $PWD/gmapsdata:/gmapsdata -p 8080:8080 gosom

Or dowload the [binary](https://github.com/gosom/google-maps-scraper/releases) for your platform and run it with the `-web` command line argument.

Note: Even if you add one keyword the results will come in at least 3 minutes. This is a minimum configured runtime


### Command line:

Expand Down
6 changes: 3 additions & 3 deletions runner/webrunner/webrunner.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,8 @@ func (w *webrunner) scrapeJob(ctx context.Context, job *web.Job) error {
allowedSeconds := max(60, len(seedJobs)*10*job.Data.Depth/50+120)

if job.Data.MaxTime > 0 {
if job.Data.MaxTime.Seconds() < 60 {
allowedSeconds = 60
if job.Data.MaxTime.Seconds() < 180 {
allowedSeconds = 180
} else {
allowedSeconds = int(job.Data.MaxTime.Seconds())
}
Expand Down Expand Up @@ -231,7 +231,7 @@ func (w *webrunner) setupMate(_ context.Context, writer io.Writer) (*scrapematea
opts := []func(*scrapemateapp.Config) error{
scrapemateapp.WithConcurrency(w.cfg.Concurrency),
scrapemateapp.WithJS(scrapemateapp.DisableImages()),
scrapemateapp.WithExitOnInactivity(time.Second * 20),
scrapemateapp.WithExitOnInactivity(time.Minute * 3),
}

csvWriter := csvwriter.NewCsvWriter(csv.NewWriter(writer))
Expand Down
4 changes: 2 additions & 2 deletions web/web.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,8 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
return
}

if maxTime < time.Minute {
http.Error(w, "max time must be more than 1m", http.StatusUnprocessableEntity)
if maxTime < time.Minute*3 {
http.Error(w, "max time must be more than 3m", http.StatusUnprocessableEntity)

return
}
Expand Down

0 comments on commit e176084

Please sign in to comment.