Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: add set zoom and coordinates features #71

Merged
merged 12 commits into from
Sep 29, 2024
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,11 @@ try `./google-maps-scraper -h` to see the command line options available:
The plugin must be a shared library (a file with .so or .dll extension).
The plugin must be compiled with the following build tags: go build -buildmode=plugin plugins/example.go.
Example: If you have your shared library in a folder /home/user/myplugins and it exposesa DummyPrinter symbol the -writer /home/user/myplugins:DummyPrinter
-zoom int
Use this to set the zoom level(0-21) for the search and MUST be use with geo parameter
-geo string
Use this to set the geo coordinates for the search and MUST be use with zoom parameter(example value '37.7749,-122.4194')

```

## Using a custom writer
Expand Down Expand Up @@ -359,4 +364,3 @@ banner is generated using OpenAI's DALE
<a href="https://www.searchapi.io/?via=gosom" rel="nofollow"> searchapi.com</a> sponsors this project via Github sponsors.

If you register via the links on my page I get a commission. This is another way to support my work

12 changes: 10 additions & 2 deletions gmaps/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type GmapJob struct {
ExtractEmail bool
}

func NewGmapJob(id, langCode, query string, maxDepth int, extractEmail bool) *GmapJob {
func NewGmapJob(id, langCode, query string, maxDepth int, extractEmail bool, geoCoordinates string, zoom int) *GmapJob {
query = url.QueryEscape(query)

const (
Expand All @@ -33,11 +33,19 @@ func NewGmapJob(id, langCode, query string, maxDepth int, extractEmail bool) *Gm
id = uuid.New().String()
}

mapURL := ""
if geoCoordinates != "" && zoom > 0 {
mapURL = fmt.Sprintf("https://www.google.com/maps/search/%s/@%s,%dz", query, strings.ReplaceAll(geoCoordinates, " ", ""), zoom)
} else {
//Warning: geo and zoom MUST be both set or not
mapURL = fmt.Sprintf("https://www.google.com/maps/search/%s", query)
}

job := GmapJob{
Job: scrapemate.Job{
ID: id,
Method: http.MethodGet,
URL: "https://www.google.com/maps/search/" + query,
URL: mapURL,
URLParams: map[string]string{"hl": langCode},
MaxRetries: maxRetries,
Priority: prio,
Expand Down
12 changes: 8 additions & 4 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func runFromLocalFile(ctx context.Context, args *arguments) error {
return err
}

seedJobs, err := createSeedJobs(args.langCode, input, args.maxDepth, args.email)
seedJobs, err := createSeedJobs(args.langCode, input, args.maxDepth, args.email, args.geoCoordinates, args.zoom)
if err != nil {
return err
}
Expand Down Expand Up @@ -220,7 +220,7 @@ func produceSeedJobs(ctx context.Context, args *arguments, provider scrapemate.J
input = f
}

jobs, err := createSeedJobs(args.langCode, input, args.maxDepth, args.email)
jobs, err := createSeedJobs(args.langCode, input, args.maxDepth, args.email, args.geoCoordinates, args.zoom)
if err != nil {
return err
}
Expand All @@ -234,7 +234,7 @@ func produceSeedJobs(ctx context.Context, args *arguments, provider scrapemate.J
return nil
}

func createSeedJobs(langCode string, r io.Reader, maxDepth int, email bool) (jobs []scrapemate.IJob, err error) {
func createSeedJobs(langCode string, r io.Reader, maxDepth int, email bool, geoCoordinates string, zoom int) (jobs []scrapemate.IJob, err error) {
scanner := bufio.NewScanner(r)

for scanner.Scan() {
Expand All @@ -250,7 +250,7 @@ func createSeedJobs(langCode string, r io.Reader, maxDepth int, email bool) (job
id = strings.TrimSpace(after)
}

jobs = append(jobs, gmaps.NewGmapJob(id, langCode, query, maxDepth, email))
jobs = append(jobs, gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom))
}

return jobs, scanner.Err()
Expand All @@ -274,6 +274,8 @@ type arguments struct {
exitOnInactivityDuration time.Duration
email bool
customWriter string
geoCoordinates string
zoom int
}

func parseArgs() (args arguments) {
Expand Down Expand Up @@ -305,6 +307,8 @@ The plugin must implement the scrapemate.ResultWriter interface.
The plugin must be a shared library (a file with .so extension).
The plugin must be compiled with the following build tags: go build -buildmode=plugin plugins/example.go.
The plugins must be placed in the same directory as the binary in a directory called plugins.`)
flag.StringVar(&args.geoCoordinates, "geo", "", "Use this to set the geo coordinates for the search and MUST be use with zoom parameter(example value '37.7749,-122.4194')")
flag.IntVar(&args.zoom, "zoom", 0, "Use this to set the zoom level(0-21) for the search and MUST be use with geo parameter")

flag.Parse()

Expand Down
Loading