Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Google I/O 2014 redeem easter egg crawler
//before running make sure you setup a GOPATH env variable and ran: "go get code.google.com/p/go.net/html"
//to run: go run ioCrawler.go -url="http://developers.google.com/"
//also try http://developer.android.com/index.html
//output goo.gl links to try and redeem will be sent to foundLinks.txt
//by the way there's an artificial "rate limit" in func crawler, you can lower that or raise it as you wish
//You can also comment out the onlyGoogleRegex code if you don't want to limit to google.com/youtube
//if you're getting I/O timeout errors, then you might need to increase the timeouts on line 231
package main
import (
"code.google.com/p/go.net/html"
"errors"
"flag"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"regexp"
"strings"
"time"
"sync"
"math/rand"
)
//prevent from downloading files we can't even read
var extToIgnore = regexp.MustCompile(`(?i)\.(gz|tar|tgz|bz2|jpg|png|jpeg|gif|psd|mp3|webm|ogg|mp4|mpeg|flv|avi|wmv|fla|mov|bin|wav|rpm|dmg|exe|zip|rar|7z|doc|ppt|pdf|iso|torrent)$`)
//if you want to index more domains feel free to update this or remove it entirely (update validateUri too to remove the check at the bottom)
var onlyGoogleRegex = regexp.MustCompile(`(?i)(google|youtube|android|golang|opers.blogspot|oogle.blogspot|youtu|angularjs|angulardart|dartlang|html5rocks)\.(com|org|net|io|be)`)
var googlRegex = regexp.MustCompile(`(?im)((?:goo\.gl/[a-zA-Z0-9]{3,9})|(?:[a-zA-Z0-9]{3,9}(?:/|\\)lg\.oog))`)
//if you're getting a lot of I/O (pun not intended...) timeouts then raise this
var defaultConnectTimeout = time.Duration(2*time.Second)
var defaultRWTimeout = time.Duration(5*time.Second)
var guessTimeout = time.Duration(1000*time.Millisecond)
var rateLimitedGoog bool
var queuedLock = sync.RWMutex{}
var knownLock = sync.RWMutex{}
var knownGooglLock = sync.RWMutex{}
var queuedGooglLock = sync.RWMutex{}
var queuedUris = map[string]bool{}
var knownUris = map[string]bool{}
var queuedGoogl = map[string]bool{}
var knownGoogl = map[string]bool{
"goo.gl/ZV63Cr": true,
"goo.gl/1nToQ4": true,
"goo.gl/q7wKp8": true,
"goo.gl/dDZIsO": true,
"goo.gl/R1jVTj": true,
"goo.gl/B92rvj": true,
"goo.gl/Au49Um": true,
"goo.gl/uQvdDQ": true,
"goo.gl/KBXDW9": true,
"goo.gl/m3uwvU": true,
"goo.gl/bhLDq0": true,
"goo.gl/gdEknO": true,
"goo.gl/8AUTRP": true,
"goo.gl/5nz6cg": true,
"goo.gl/T3mzjk": true,
"goo.gl/5FwCJy": true,
"goo.gl/3QGGK0": true,
"goo.gl/K6E7l8": true,
"goo.gl/xZM9jt": true,
"goo.gl/rMFYNX": true,
"goo.gl/F3noUV": true,
"goo.gl/JYSlLQ": true,
"goo.gl/hmgbrW": true,
"goo.gl/0gNnSE": true,
"goo.gl/Tw6kP9": true,
"goo.gl/RB2zhx": true,
"goo.gl/kJPqT3": true,
"goo.gl/qllLsG": true,
"goo.gl/zj9bO0": true,
"goo.gl/bjAPLg": true,
"goo.gl/7tKTyd": true,
"goo.gl/eXeVao": true,
"goo.gl/Dpoyqi": true,
"goo.gl/iztHMd": true,
"goo.gl/HACUad": true,
"goo.gl/oJDnYs": true,
"goo.gl/ia7jLd": true,
"goo.gl/AJRPcI": true,
"goo.gl/bS7lhD": true,
"goo.gl/lyBKa0": true,
"goo.gl/OmM6DN": true,
"goo.gl/puUsyX": true,
"goo.gl/Gc8Dsd": true,
"goo.gl/P3cEUu": true,
"goo.gl/9lT4ZV": true,
"goo.gl/HNA9YG": true,
"goo.gl/xjP4Ke": true,
"goo.gl/ykmzUC": true,
"goo.gl/EOiis9": true,
"goo.gl/wgzU7N": true,
"goo.gl/5ixjt4": true,
"goo.gl/L2N5sY": true,
"goo.gl/SWNo61": true,
"goo.gl/06Hov3": true,
"goo.gl/JOzJ9j": true,
"goo.gl/8JoXn0": true,
"goo.gl/y1y9Lh": true,
"goo.gl/FDTdEx": true,
"goo.gl/LgqaL0": true,
"goo.gl/OQwgds": true,
"goo.gl/hZWUau": true,
"goo.gl/wfkUr7": true,
"goo.gl/sK6kAA": true,
"goo.gl/DpYUwO": true,
"goo.gl/vLppsl": true,
"goo.gl/KVDTVy": true,
"goo.gl/dGrMwY": true,
"goo.gl/YOTeO1": true,
"goo.gl/PltckZ": true,
"goo.gl/sUJ1r7": true,
"goo.gl/okXi4W": true,
"goo.gl/ENWVvN": true,
"goo.gl/BcLOM9": true,
"goo.gl/R9MgF3": true,
"goo.gl/egnhgc": true,
"goo.gl/Jca6KP": true,
"goo.gl/1AiXEk": true,
"goo.gl/iMmkIj": true,
"goo.gl/E1vTJk": true,
"goo.gl/pikTTi": true,
"goo.gl/9KK0k0": true,
"goo.gl/Oof8mG": true,
"goo.gl/X1R54k": true,
"goo.gl/ykH2OP": true,
"goo.gl/cuSgmK": true,
"goo.gl/I6Lpbz": true,
"goo.gl/JFKDGw": true,
"goo.gl/o0YxL2": true,
"goo.gl/cCpQrU": true,
"goo.gl/sWHuKQ": true,
"goo.gl/FhGPJW": true,
"goo.gl/v6jFEL": true,
"goo.gl/hBKyB5": true,
"goo.gl/mH9rXs": true,
"goo.gl/3QXZ9r": true,
"goo.gl/6M31VD": true,
"goo.gl/ZaQQyr": true,
"goo.gl/FbPhW1": true,
"goo.gl/EePxH9": true,
"goo.gl/LLy4UO": true,
"goo.gl/yGQsaV": true,
"goo.gl/MtXamT": true,
"goo.gl/SaKrk1": true,
"goo.gl/Jtc6vr": true,
"goo.gl/UkoouX": true,
"goo.gl/JUAYZY": true,
"goo.gl/1ZTcPQ": true,
"goo.gl/IUJ41u": true,
"goo.gl/3A5sUu": true,
"goo.gl/bPPmeA": true,
"goo.gl/KF857p": true,
"goo.gl/PrJELw": true,
"goo.gl/TDxlVd": true,
"goo.gl/B9W5Mm": true,
"goo.gl/m3JCoz": true,
"goo.gl/ZObZyF": true,
"goo.gl/aQ1W4k": true,
"goo.gl/IpXSvB": true,
"goo.gl/dBPRBg": true,
"goo.gl/s1HvIH": true,
"goo.gl/ttepYl": true,
"goo.gl/IWa8N2": true,
"goo.gl/Rduuii": true,
"goo.gl/rvhMqc": true,
"goo.gl/DJi6x0": true,
"goo.gl/nEj7wW": true,
"goo.gl/eFJLBs": true,
"goo.gl/7jXWOU": true,
"goo.gl/KiYy3o": true,
"goo.gl/wS3Ari": true,
"goo.gl/HH7XXz": true,
"goo.gl/txZSBa": true,
"goo.gl/fIabRh": true,
"goo.gl/iu38dw": true,
"goo.gl/qa4Frv": true,
"goo.gl/1QTi2T": true,
"goo.gl/Ql8Ap8": true,
"goo.gl/pXX3oV": true,
"goo.gl/eFJLB": true,
}
var uriQueue = make(chan *url.URL, 16384)
var googlQueue = make(chan *url.URL, 4096)
var googlOutputFile *os.File
var shouldGuess bool
func main() {
startURL := flag.String("url", "", "the url to start crawling")
guess := flag.Bool("guess", false, "guess goo.gl links")
flag.Parse()
shouldGuess = *guess
startUri, err := url.Parse(*startURL)
if err != nil || *startURL == "" {
fmt.Printf("Failed to parse starting URL: %s. Response: %s\n", *startURL, err)
return
}
googlOutputFile, err = os.OpenFile("./foundLinks.txt", os.O_APPEND | os.O_CREATE | os.O_WRONLY, 0644)
if err != nil {
fmt.Printf("Failed to open foundLinks.txt. Error: %s\n", err)
return
}
defer func() {
if err := googlOutputFile.Close(); err != nil {
fmt.Printf("Failed to close output file: %s\n", err)
}
}()
for i := 0; i < 48; i++ {
go crawler()
}
for i := 0; i < 8; i++ {
go googlChecker()
}
for i := 0; i < 1; i++ {
go guesser()
}
//fakeGoogl, _ := url.Parse("http://goo.gl/ZV63Cr")
//googlQueue <- fakeGoogl
uriQueue <- startUri
waitForSignal()
}
func crawler() {
var nextUri *url.URL
for {
<-time.After(500*time.Millisecond)
select {
case nextUri = <-uriQueue:
crawl(nextUri)
}
}
}
func googlChecker() {
var nextUri *url.URL
for {
if rateLimitedGoog {
fmt.Println("RATE LIMITED!! Waiting...")
<-time.After(120*time.Second)
}
select {
case nextUri = <-googlQueue:
verifyGoogl(nextUri)
}
}
}
func normalizeUri(uri *url.URL) string {
host := uri.Host
//remove www from the host
if len(host) > 4 && host[0:4] == "www." {
host = host[4:]
}
path := uri.Path
if path == "" || path == "/" {
return host
}
pathLen := len(path)
//remove any trailing slashes
if path[pathLen-1] == '/' {
path = path[:pathLen-1]
}
//unfortunately including RawQuery in here to fix youtube links not working anymore
return strings.Join([]string{host, path, uri.RawQuery}, "")
}
func validateUri(uri *url.URL) (bool, string) {
if uri.Scheme != "http" && uri.Scheme != "https" {
return false, ""
}
normalUri := normalizeUri(uri)
knownLock.RLock()
if _, ok := knownUris[normalUri]; ok {
knownLock.RUnlock()
//already checked this
return false, normalUri
}
knownLock.RUnlock()
if uri.Host == "" {
return false, normalUri
}
if extToIgnore.MatchString(uri.Path) {
return false, normalUri
}
if !onlyGoogleRegex.MatchString(uri.Host) {
return false, normalUri
}
return true, normalUri
}
//from http://stackoverflow.com/questions/16895294/how-to-set-timeout-for-http-get-requests-in-golang
func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
return func(netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, cTimeout)
if err != nil {
return nil, err
}
conn.SetDeadline(time.Now().Add(rwTimeout))
return conn, nil
}
}
func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration, baseNormalUri string) *http.Client {
return &http.Client{
Transport: &http.Transport{
Dial: TimeoutDialer(connectTimeout, readWriteTimeout),
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
valid, newNormalUri := validateUri(req.URL)
//allow redirects to the *same* url since we might already have this url in knownUris because we're processing it right now
if !valid && baseNormalUri != newNormalUri {
return errors.New("Redirect:Invalid")
}
return nil
},
}
}
func crawl(uri *url.URL) {
valid, normalUri := validateUri(uri)
if !valid || uri == nil {
queuedLock.Lock()
delete(queuedUris, normalUri)
queuedLock.Unlock()
return
}
knownLock.Lock()
knownUris[normalUri] = true
knownLock.Unlock()
queuedLock.Lock()
delete(queuedUris, normalUri)
queuedLock.Unlock()
findYoutubeAnnotations(uri)
urlStr := uri.String()
fmt.Printf("Fetching: %s\n", urlStr)
client := NewTimeoutClient(defaultConnectTimeout, defaultRWTimeout, normalUri)
resp, err := client.Get(urlStr)
if err != nil {
var errMsg string
urlErr, ok := err.(*url.Error)
if !ok {
errMsg = err.Error()
} else {
errMsg = urlErr.Err.Error()
}
if errMsg != "Redirect:Invalid" {
fmt.Printf("Failed to Get URL: %s. Response: %s\n", urlStr, errMsg)
}
knownLock.Lock()
delete(knownUris, normalUri)
knownLock.Unlock()
return
}
htmlBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Printf("Failed to read from server to get full HTML for %s. Error: %s\n", urlStr, err)
knownLock.Lock()
delete(knownUris, normalUri)
knownLock.Unlock()
return
}
resp.Body.Close()
htmlStr := string(htmlBytes)
locateAndSaveGoogl(htmlStr)
doc, err := html.Parse(strings.NewReader(htmlStr))
if err != nil {
fmt.Printf("Failed to parse HTML for %s. Error: %s\n", urlStr, err)
return
}
findAnchors(doc, uri)
}
func findAnchors(node *html.Node, baseUri *url.URL) {
if node.Type == html.ElementNode && (node.Data == "a" || node.Data == "url") {
for _, attr := range node.Attr {
if (node.Data == "a" && attr.Key == "href") || (node.Data == "url" && attr.Key == "value") {
newUri, err := url.Parse(attr.Val)
if err != nil || newUri == nil {
fmt.Printf("Invalid url detected: %s. Error %s\n", attr.Val, err)
break
}
if !newUri.IsAbs() {
newUri = baseUri.ResolveReference(newUri)
}
valid, newNorm := validateUri(newUri)
if valid {
//fmt.Printf("Found new URL! %s\n", newUri.String())
queuedLock.Lock()
if _, ok := queuedUris[newNorm]; !ok {
queuedUris[newNorm] = true
queuedLock.Unlock()
select {
case uriQueue <- newUri:
case <-time.After(time.Second * 1):
//fmt.Println("Queue is full!!")
return
}
} else {
queuedLock.Unlock()
}
}
break
}
}
} else if node.Type == html.ElementNode && node.Data == "head" {
//skip looping through head
node = node.NextSibling
if node == nil {
return
}
}
for nextNode := node.FirstChild; nextNode != nil; nextNode = nextNode.NextSibling {
findAnchors(nextNode, baseUri)
}
}
func findYoutubeAnnotations(uri *url.URL) {
if (uri.Host == "youtube.com" || uri.Host == "www.youtube.com") {
if len(uri.Path) < 6 || uri.Path[0:6] != "/watch" {
return
}
queryVals := uri.Query();
if queryVals == nil {
return
}
videoIDs, ok := queryVals["v"]
if !ok || len(videoIDs) < 1 {
videoIDsCommas, ok := queryVals["video_ids"]
if !ok || len(videoIDsCommas) < 1 {
return
}
videoIDs = strings.Split(videoIDsCommas[0], ",")
}
for _, videoID := range videoIDs {
addYoutubeAnnotations(videoID)
}
} else if (uri.Host == "youtu.be") {
addYoutubeAnnotations(uri.Path);
} else {
return
}
}
//thanks https://plus.google.com/+MarcLesterTan
func addYoutubeAnnotations(videoID string) {
if len(videoID) < 5 {
return
}
newUri, err := url.Parse("https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=" + videoID)
if err != nil || newUri == nil {
fmt.Printf("Invalid youtube annotations url generated. Error %s\n", err)
return
}
valid, newNorm := validateUri(newUri)
if valid {
//fmt.Printf("Added new youtube annotations URL! %s\n", newUri.String())
queuedLock.Lock()
if _, ok := queuedUris[newNorm]; !ok {
queuedUris[newNorm] = true
queuedLock.Unlock()
select {
case uriQueue <- newUri:
case <-time.After(time.Second * 1):
//fmt.Println("Queue is full!!")
return
}
} else {
queuedLock.Unlock()
}
}
}
func ReverseString(str string) string {
i := len(str)
final := make([]rune, i)
for _, r := range str {
i--
final[i] = r
}
return string(final)
}
func locateAndSaveGoogl(htmlStr string) {
matches := googlRegex.FindAllString(htmlStr, -1)
if matches == nil {
return
}
for _, match := range matches {
//detect backwards goo.gl links
if len(match) > 6 && match[len(match)-6:] == "lg.oog" {
match = ReverseString(match)
}
//fmt.Printf("Found goo.gl link!: %s\n", match)
//prevent backwards slashes from breaking this
uri, err := url.Parse("http://" + strings.Replace(match, `\`, "/", -1))
if err != nil {
fmt.Printf("Failed to parse goo.gl link: %s. Error: %s\n", match, err)
continue
}
normalUri := normalizeUri(uri)
knownGooglLock.RLock()
if _, ok := knownGoogl[normalUri]; ok {
knownGooglLock.RUnlock()
continue
}
knownGooglLock.RUnlock()
queuedGooglLock.Lock()
if _, ok := queuedGoogl[normalUri]; ok {
queuedGooglLock.Unlock()
continue
}
queuedGoogl[normalUri] = true
queuedGooglLock.Unlock()
googlQueue <- uri
}
}
var validGooglChars = "avE1k4yiQhAzl9B6DpIRdCqnF8tbSr7J2NKGT3VmoMPuc5UHeLjOZWxfwXgsY0"
func guesser() { //s float64
if !shouldGuess {
return
}
var i int
n := make([]uint8, 6)
for {
if rateLimitedGoog {
<-time.After(180*time.Second)
continue
}
i = 0
for i < 6 {
n[i] = validGooglChars[int(rand.Float64() * 62)]
i++
}
normal := "goo.gl/" + string(n)
knownGooglLock.RLock()
if _, ok := knownGoogl[normal]; ok {
knownGooglLock.RUnlock()
continue
}
knownGooglLock.RUnlock()
queuedGooglLock.Lock()
if _, ok := queuedGoogl[normal]; ok {
queuedGooglLock.Unlock()
continue
}
queuedGoogl[normal] = true
queuedGooglLock.Unlock()
uri, err := url.Parse("http://" + normal)
if err != nil {
fmt.Printf("Failed to generate valid goo.gl link: %s. Error: %s\n", normal, err)
continue
}
if guessTimeout > 0 {
<-time.After(guessTimeout)
}
googlQueue <- uri
}
}
var googlClient = &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// /events/io/2014/redeem
//yes... this is ghetto how it works
if req.URL.Host == "developers.google.com" && len(req.URL.Path) > 22 && req.URL.Path[0:22] == "/events/io/2014/redeem" {
return errors.New("Redirect:Valid")
}
return errors.New("Redirect:Invalid")
},
}
func verifyGoogl(uri *url.URL) {
normalUri := normalizeUri(uri)
queuedGooglLock.Lock()
delete(queuedGoogl, normalUri)
queuedGooglLock.Unlock()
knownGooglLock.Lock()
if _, ok := knownGoogl[normalUri]; ok {
knownGooglLock.Unlock()
return
}
knownGoogl[normalUri] = true
knownGooglLock.Unlock()
urlStr := uri.String()
fmt.Println("Fetching goo.gl link: " + urlStr)
resp, err := googlClient.Head(urlStr)
if err != nil {
var errMsg string
urlErr, ok := err.(*url.Error)
if !ok {
errMsg = err.Error()
} else {
errMsg = urlErr.Err.Error()
}
if errMsg == "Redirect:Valid" {
rateLimitedGoog = false
fmt.Printf("Found a valid goo.gl redirect!! %s\n", urlStr)
if _, err := googlOutputFile.WriteString(urlStr + "\n"); err != nil {
fmt.Printf("ERROR! Failed to write goo.gl redirect to output file! Error: %s/n", err)
}
} else if errMsg == "Redirect:Invalid" {
rateLimitedGoog = false
//do nothing
} else {
fmt.Printf("Failed to Head URL: %s. Response: %s\n", urlStr, errMsg)
knownGooglLock.Lock()
delete(knownGoogl, normalUri)
knownGooglLock.Unlock()
}
}
if resp.StatusCode == 403 {
rateLimitedGoog = true
} else {
rateLimitedGoog = false
}
}
func waitForSignal() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
<-c
fmt.Println("Received SIGINT!")
}
@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 17, 2014

Updated

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Thanks! Updated the list and the script.

@kvcrawford

This comment has been minimized.

Copy link

commented Apr 18, 2014

http://goo.gl/sK6kAA

I've been spidering all day, with no luck :(

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Thanks @kvcrawford! I also just fixed a bug that prevented youtube annotations from being crawled correctly so make sure to update your local copies!!

@sadovnychyi, I added video annotation searching in revision 18 and fixed a bug with them in revision 20. I also just added support for videoID lists in playlist links.

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

goo.gl/sUJ1r7 -- from image https://developers.google.com/appengine/docs/images/modules_yaml_files2.png
probably it's worth to download all images from docs. doing it.

@fbballan93

This comment has been minimized.

Copy link

commented Apr 18, 2014

can please sombody tell me how to deal with that and how can i run it on my mac

@sadovnychyi

This comment has been minimized.

@schwiz

This comment has been minimized.

Copy link

commented Apr 18, 2014

If this gets me in IO I'm totally buying you a beer!

@ljbotero

This comment has been minimized.

Copy link

commented Apr 18, 2014

It's been about 4 hours I'be been crawling the whole domain and have found a few, but all used... anyone! please help!

@bburky

This comment has been minimized.

Copy link

commented Apr 18, 2014

$ find /Applications/Android\ Studio.app/sdk/docs/images -type f | sed 's_/Applications/Android\ Studio.app/sdk/docs/images_http://developer.android.com/images_' | wget -x -i -
$ diff -r developer.android.com/images /Applications/Android\ Studio.app/sdk/docs/images
Binary files developer.android.com/images/admin/device-admin-activate-prompt.png and /Applications/Android Studio.app/sdk/docs/images/admin/device-admin-activate-prompt.png differ
[...]

Sadly, they're all gone.

@trogdan

This comment has been minimized.

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

KVDTVy
dGrMwY
YOTeO1
PltckZ

Extracted from images. All gone :(

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Remember to keep updating your scripts guys! Just fixed some bugs earlier and added more domains to the whitelist!

Thanks @trogdan and @sadovnychyi

@greenrift

This comment has been minimized.

Copy link

commented Apr 18, 2014

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

goo.gl/Jca6KP

@greenrift

This comment has been minimized.

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Updated with the codes mentioned above.

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

They said they're going to be recycling the used codes out, so maybe again throughout tomorrow.

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Just updated to support backwards lg.oog links

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

https://gist.github.com/sadovnychyi/11025054 -- image downloader if someone interested. Found a lot of codes, but all gone.

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

http://goo.gl/pikTTi
http://goo.gl/9KK0k0
http://goo.gl/Oof8mG

are from the hidden digests on developers.google.com

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

Can someone understand characters here?
http://imagehost4.online-image-editor.com/oie_upload/images/OIE_2013_2972597_59494793/187251vNdL5dWf.png
http://imagehost4.online-image-editor.com/oie_upload/images/187141Nv4Nh2/187118cKWwKzv8.png
I've processed 1100 images. No luck!

Almost done writing a script to monitor for image updates on developers.google.com

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

@sadovnychyi Those links don't work.

@sadovnychyi

This comment has been minimized.

Copy link

commented Apr 18, 2014

EswtC4h5UKXQVcClfN/r3QfzS4A=

base64 -> �Ì-��yP¥ÐUÀ¥|ßëÝ�óK�

Found it on developers.
Any ideas?

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

See my comments on Google+:
https://plus.google.com/u/0/+MikeJarvisW/posts/Txizoa5uECd

I explain how to get the sha1 and then the result of the brute force.

@Freesoul1982

This comment has been minimized.

Copy link

commented Apr 18, 2014

Another used one
http://goo.gl/JFKDGw

@greenrift

This comment has been minimized.

Copy link

commented Apr 18, 2014

@chrisgco

This comment has been minimized.

Copy link

commented Apr 18, 2014

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

http://goo.gl/cuSgmK
http://goo.gl/I6Lpbz

Updated with these above and the ones that @greenrift, @Freesoul1982, @bk9, @digitalfiz, and @chrisgrants brought up. Thanks!

@bka9

This comment has been minimized.

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

Thanks @bka9. Updated!

@kageiit

This comment has been minimized.

@pitecus

This comment has been minimized.

Copy link

commented Apr 18, 2014

Used: #mimimi

http://goo.gl/4M8XP4

@cipriani14

This comment has been minimized.

Copy link

commented Apr 18, 2014

Been running it for about 3 hours now, with nothing found. Bleh

But thanks for putting this together.. much appreciated!

@chrisgco

This comment has been minimized.

Copy link

commented Apr 18, 2014

@dpicky

This comment has been minimized.

Copy link

commented Apr 18, 2014

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 18, 2014

I'm not around a computer but thanks for updating with all the comments guys! Also, I recommend restarting the script every few hours since it keeps track of scanned pages. If Google updates a page AFTER you started crawling you might never rescan it. I had planned on fixing this bug but I haven't had time.

@cipriani14

This comment has been minimized.

Copy link

commented Apr 18, 2014

@CharlesCai930

This comment has been minimized.

@w4tson

This comment has been minimized.

Copy link

commented Apr 19, 2014

All the images I found yesterday have had the codes removed. However just found this http://goo.gl/1QTi2T so maybe there's bunch been released again?

@JaredBanyard

This comment has been minimized.

Copy link

commented Apr 19, 2014

Wish I had found this sooner, running now...

@cipriani14

This comment has been minimized.

Copy link

commented Apr 19, 2014

Are you guys finding any unused links at all? Anything I find is already used...

@davissp14

This comment has been minimized.

Copy link

commented Apr 19, 2014

No luck so far...

@thegreyspot

This comment has been minimized.

Copy link

commented Apr 19, 2014

@shantanuthatte

This comment has been minimized.

Copy link

commented Apr 19, 2014

Scouring for hours now.... Really slow reveal rate.. Even after that, most of them are dead/used... Hopefully I'll get something...

@JaredBanyard

This comment has been minimized.

Copy link

commented Apr 21, 2014

Pretty sure its done..

@fastest963

This comment has been minimized.

Copy link
Owner Author

commented Apr 21, 2014

Updated with all the latest codes from this weekend.

@momalave

This comment has been minimized.

Copy link

commented Apr 21, 2014

Searched for hours this weekend and just got used codes. Are you guys still finding any or have an extra code that someone could PM me? A friend of mine got an email saying that they would be charged (I'm assuming they got a ticket in the lottery), it's a bummer that I can't join them.

@vovkab

This comment has been minimized.

Copy link

commented Apr 21, 2014

No luck for me too, all used :(

@metc

This comment has been minimized.

Copy link

commented Apr 22, 2014

Thanks for sharing, but unfortunately they are all used. I think it is done now...

@NicolasRannou

This comment has been minimized.

Copy link

commented Apr 22, 2014

if links are valid, they should show up in the found txt file?

@jakir678

This comment has been minimized.

Copy link

commented Apr 22, 2014

Hey James, so I found a few more of the goo.gl URLs after scouring the developers.google.com site. Unfortunately all were taken. Also after doing some research - it seems that mostly all of these links were created on March 14, or 22. Also they are placed in videos and pages that only go as far back as 6 months from today.

You can tell if a link has been taken by checking the stats when adding ".info" after the URL. For example: "goo.gl/XS2wZh.info" shows that this link has been viewed 60 times. (at the time of this post).

I'm pretty sure Google has another set of codes waiting to be distributed. Let me know your thoughts.

@csobrinho

This comment has been minimized.

Copy link

commented Apr 22, 2014

Any luck?

@vovkab

This comment has been minimized.

@sachins

This comment has been minimized.

Copy link

commented Apr 23, 2014

Came across http://goo.gl/6K9bLw but seems like already used.

@grahammccarthy

This comment has been minimized.

@vovkab

This comment has been minimized.

Copy link

commented Apr 25, 2014

goo.gl/RiEzSX

@sachins

This comment has been minimized.

Copy link

commented Apr 25, 2014

@daytonpruet

This comment has been minimized.

Copy link

commented Feb 10, 2015

To get this script to work with the latest version of GO, you need to replace the URL in line 14 "code.google.com/p/go.net/html" with "golang.org/x/net/html".

@shafty023

This comment has been minimized.

Copy link

commented Mar 17, 2015

I wonder if this will be relevant for 2015?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.