Skip to content

Instantly share code, notes, and snippets.

@jexp
Last active August 29, 2015 14:04
Show Gist options
  • Save jexp/3b60a3a1f29061853e8c to your computer and use it in GitHub Desktop.
Save jexp/3b60a3a1f29061853e8c to your computer and use it in GitHub Desktop.
Simple Ruby script to pull tweets from Twitter into Neo4j using Cypher
BEARER='...'
def load_tweets(query,since_id=nil,lang="en",page=1,rpp=100)
res=RestClient.get('https://api.twitter.com/1.1/search/tweets.json',
{:params=> {:q=>query, :lang=>lang,:count=>rpp,:result_type=>:recent,:since_id=>since_id},
:accept=>:json,
:Authorization => "Bearer #{BEARER}"})
puts "query '#{query}'\n since id #{since_id} result #{res.code}"
return [] unless res.code==200
data=JSON.parse(res.to_str)
data['statuses']
end
add_tweet_query = "
UNWIND {tweets} as t
WITH t, t.entities as e, t.user as u
MERGE (tweet:Tweet {id:t.id})
ON CREATE SET tweet.text = t.text, tweet.created_at = t.created_at,tweet.short = t.short
MERGE (user:User {screen_name:u.screen_name})
ON CREATE SET user.name = u.name,user.name = u.name, user.location = u.location, user.profile_image_url = u.profile_image_url,
user.followers_count = u.followers_count, user.friends_count = u.friends_count, user.statuses_count = u.statuses_count
MERGE (user)-[:POSTED]->(tweet)
FOREACH (h IN e.hashtags |
MERGE (tag:Tag {name:h.text}) MERGE (tag)<-[:TAGGED]-(tweet))
FOREACH (u IN e.urls |
MERGE (url:Url {url:u.expanded_url}) MERGE (url)<-[:CONTAINS]-(tweet))
FOREACH (m IN e.user_mentions |
MERGE (mentioned:User {screen_name:m.screen_name}) ON CREATE SET mentioned.name = m.name MERGE (mentioned)<-[:MENTIONS]-(tweet))
FOREACH (r IN [r in [t.in_reply_to_status_id] WHERE r IS NOT NULL] |
MERGE (reply:Tweet {id:r}) MERGE (tweet)-[:REPLY_TO]->(reply))
FOREACH (sn IN [sn in [t.in_reply_to_screen_name] WHERE sn IS NOT NULL] |
MERGE (replied:User {screen_name:sn}) MERGE (tweet)-[:REPLY_TO_USER]->(replied))
RETURN count(*) as added_tweets
"
neo = Neography::Rest.new
[[:t, :Tweet, :id],[:u, :User, :screen_name],[:u, :Url, :url],[:t, :Tag, :name]]
.each do |(id,label,prop)|
neo.commit_transaction("create constraint on (#{id}:#{label}) assert #{id}.#{prop} is unique")
end
while true
twitter_id = nil
begin
query = ENV['TWITTER_SEARCH']||'oscon OR oreillymedia OR graphalchemist OR graphconnect OR neo4j'
tweets=load_tweets(query,twitter_id)
twitter_id = tweets.first["id"]
puts "Found #{tweets.length} new tweets"
tweets.each { |tweet| tweet["short"] = tweet["text"].gsub(/https?:\S+/i,"(link)").split(/\s/)[0..10].join(" ")+" ..." }
res = neo.commit_transaction([add_tweet_query,{:tweets=>tweets}])
puts res
rescue => e
puts e
end
sleep(300)
end
@jexp
Copy link
Author

jexp commented Jul 16, 2014

It looks like this:

@jexp
Copy link
Author

jexp commented Jul 16, 2014

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment