Skip to content

Instantly share code, notes, and snippets.

@kjk kjk/chromedp_bug.go
Created Jul 24, 2019

Embed
What would you like to do?
chromedp bug
package main
import (
"context"
"fmt"
"net/url"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp"
)
// $$("div.rc div.r").forEach((node) => console.log($("a", node).href))
// $$("div.rc > div:first-child > a:first-child").forEach((x) => console.log(x.href))
// $x('//div[@class="rc"]//div[@class="r"]//a[1]')
func spiderGoogle(searchTerm string) ([]string, error) {
//opts := append([]chromedp.ExecAllocatorOption{}, defaultOpts...)
//chromePath := `C:\Users\kjk\AppData\Local\documentalist\data\chrome-win\chrome.exe`
//opts = append(opts, chromedp.ExecPath(chromePath))
opts := chromedp.DefaultExecAllocatorOptions
ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
ctx, cancel = chromedp.NewContext(
ctx,
//chromedp.WithLogf(fmtPrintf),
)
defer cancel()
// create a timeout
ctx, cancel = context.WithTimeout(ctx, 10*time.Second)
defer cancel()
qe := url.QueryEscape(searchTerm)
uri := fmt.Sprintf("https://www.google.com/search?q=" + qe)
fmt.Printf("before run, uri='%s'\n", uri)
sel := "div.rc > div:first-child > a:first-child"
var nodes []*cdp.Node
err := chromedp.Run(ctx,
chromedp.Navigate(uri),
// chromedp.WaitVisible(`foot`, chromedp.ByID),
chromedp.WaitVisible("#foot"),
chromedp.Nodes(sel, &nodes),
)
if err != nil {
return nil, err
}
var res []string
for _, node := range nodes {
href := node.AttributeValue("href")
res = append(res, href)
}
return res, nil
}
func main() {
res, err := spiderGoogle("krzysztof kowalczyk")
if err != nil {
fmt.Printf("spiderGoogle() failed with %s\n", err)
} else {
fmt.Printf("spiderGoogle returned %d results\n", len(res))
for _, s := range res {
fmt.Printf("%s\n", s)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.