Skip to content

Instantly share code, notes, and snippets.

@EdmundMartin
Created November 5, 2017 14:55
Show Gist options
  • Save EdmundMartin/09b795502493b6229374bc0a6c686e8e to your computer and use it in GitHub Desktop.
Save EdmundMartin/09b795502493b6229374bc0a6c686e8e to your computer and use it in GitHub Desktop.
package main
import (
"fmt"
"math/rand"
"net/http"
"net/url"
"time"
)
func RandomString(options []string) string {
rand.Seed(time.Now().Unix())
randNum := rand.Int() % len(options)
return options[randNum]
}
func scrapeClient(targetURL string, proxyOptions []string, userAgents []string) (*http.Response, error) {
selectedProxy := RandomString(proxyOptions)
proxyUrl, _ := url.Parse(selectedProxy)
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
req, _ := http.NewRequest("GET", targetURL, nil)
req.Header.Set("User-Agent", RandomString(userAgents))
res, err := client.Do(req)
if err != nil {
return nil, err
} else {
return res, nil
}
}
var userAgents = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
}
var proxies = []string{
"http://138.68.150.51:8118",
"http://82.206.132.106:80",
"http://178.62.88.110:8118",
}
func main() {
res, _ := scrapeClient("http://edmundmartin.com", proxies, userAgents)
fmt.Println(res)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment