Created
October 25, 2016 03:35
-
-
Save ScottMansfield/7cadaf29c16b6f68d653c2efecc218d4 to your computer and use it in GitHub Desktop.
Twitter search utility
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"flag" | |
"fmt" | |
"log" | |
"net/url" | |
"os" | |
"os/signal" | |
"regexp" | |
"strings" | |
"github.com/ChimeraCoder/anaconda" | |
) | |
var stream *anaconda.Stream | |
func init() { | |
sigs := make(chan os.Signal) | |
signal.Notify(sigs, os.Interrupt) | |
go func() { | |
<-sigs | |
stream.Stop() | |
}() | |
} | |
func main() { | |
var track string | |
var match string | |
var unique bool | |
flag.StringVar(&track, "track", "", "String to track on the twitter stream API. Used directly. E.g. 'it's 2016,its 2016'. See: https://dev.twitter.com/streaming/overview/request-parameters#track") | |
flag.StringVar(&match, "match", "", "Regex to match against. Tweets are lowercased and stripped to [0-9a-z-_] for match, so capital letters will fail to match.") | |
flag.BoolVar(&unique, "unique", false, "Removes duplicate tweets from output. Helpful if there's tons of retweets.") | |
flag.Parse() | |
if track == "" || match == "" { | |
fmt.Println("Both --track and --match must be specified.") | |
fmt.Printf("Track: %v\nMatch: %v\n", track, match) | |
flag.PrintDefaults() | |
os.Exit(-1) | |
} | |
// map to hold duplicates | |
seen := make(map[string]struct{}) | |
// Set up regex used for cleaning tweets | |
reg := regexp.MustCompile("[^a-zA-Z0-9 _-]") | |
// The match regex must compile or panic | |
matchreg := regexp.MustCompile(match) | |
anaconda.SetConsumerKey("consumerkey") | |
anaconda.SetConsumerSecret("consumersecret") | |
api := anaconda.NewTwitterApi("readthe", "documentation") | |
v := url.Values{} | |
v.Add("track", track) | |
stream = api.PublicStreamFilter(v) | |
// keep a count of processed messages | |
count := 0 | |
for o := range stream.C { | |
count++ | |
if count % 100 == 0 { | |
log.Println("Count:", count) | |
} | |
switch o := o.(type) { | |
case anaconda.Tweet: | |
text := reg.ReplaceAllLiteralString(o.Text, "") | |
text = strings.ToLower(text) | |
if unique { | |
if _, ok := seen[text]; !ok && matchreg.MatchString(text) { | |
log.Println(o.Text) | |
seen[text] = struct{}{} | |
} | |
} else if matchreg.MatchString(text) { | |
log.Println(o.Text) | |
} | |
case anaconda.StatusDeletionNotice: | |
//log.Printf("Status deleted. ID: %d\n", o.Id) | |
default: | |
log.Printf("Unrecognized message: %#v\n", o) | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment