Skip to content

Instantly share code, notes, and snippets.

@llonchj
Created October 17, 2018 21:46
Show Gist options
  • Save llonchj/da8b782c06c7ca45744e3af66753133b to your computer and use it in GitHub Desktop.
Save llonchj/da8b782c06c7ca45744e3af66753133b to your computer and use it in GitHub Desktop.
Proof of concept
package main
import (
"fmt"
"net/http"
"github.com/gocolly/colly"
)
type MyCollector struct {
*colly.Collector
}
func NewMyCollector(collector *colly.Collector) *MyCollector {
if collector == nil {
collector = colly.NewCollector()
}
c := MyCollector{
Collector: collector.Clone(),
}
return &c
}
func main() {
URL := "http://www.gmail.com/"
c := NewMyCollector(nil)
c.ParseHTTPErrorResponse = true
c.RedirectHandler = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
c.OnResponse(func(r *colly.Response) {
if r.StatusCode == 200 {
panic(fmt.Errorf("expected 301, got %d - %s", r.StatusCode, r.Request.URL.String()))
}
})
c.OnError(func(r *colly.Response, err error) {
fmt.Println("error", err, r.StatusCode, r.Request.URL.String())
})
c.Visit(URL)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment