mort (owner)

Fork Of

gist: 223155 by guillermo Twitter Backup

Revisions

gist: 224913 Download_button fork
public
Description:
Tweaking Guillermo's original script to add support for accounts with a huge amount of tweets ;)
Public Clone URL: git://gist.github.com/224913.git
Embed All Files: show embed
twitter_backup.rb #
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
require 'twitter'
 
 
# TwitterBackup.new(user,pass).backup!
# Will save tweets to files
# ~/.twitter_backup/tweets/tweet_id.yaml # Full yaml tweet
# ~/.twitter_backup/tweets/tweet_id.txt # Only tweet text
 
# Search locally in your tweets
# cat ~/.twitter_backup/tweets/*.txt | grep -i 'some text'
 
 
class TwitterBackup
  
  def initialize(user,pass)
    @user,@pass = user, pass
    @twitter = Twitter::Base.new(Twitter::HTTPAuth.new(@user,@pass))
  end
 
  def total
    @twitter.verify_credentials.statuses_count
  end
  
  def remaining_hits
    @twitter.rate_limit_status.remaining_hits
  end
  
  def backup!
    puts "Backing up #{total} tweets for #{@user}"
    mkdir
    page = start_page
    res = [1,2,3]
    while (res.size > 0)
      puts "Fetching page #{page}"
      begin
        res = @twitter.user_timeline(:page => page)
      rescue Twitter::RateLimitExceeded
        puts "Twitter API exception: rate limit exceeded"
        save_latest_page(page)
        exit
      end
      res.each do |tweet|
        puts "Processing tweet #{tweet.id}: #{tweet.text}"
        save(tweet)
      end
      page += 1
    end
  end
 
private
 
  def save(tweet)
    write_yaml(tweet.id,tweet)
    write_txt(tweet.id,tweet.text)
  end
  
  def save_latest_page(page)
    f = File.open(backup_dir+"/.latest_page",'w')
    f.write page
    f.close
  end
  
  def start_page
    f_path = backup_dir+"/.latest_page"
    
    n = if File.exists?(f_path)
      f = File.open(f_path, 'r')
      v = f.readline
      f.close
      File.unlink(f_path)
      v
    else
      1
    end
    
    n
  end
  
  def write_yaml(id,data)
    f = File.open(backup_dir+"/#{id.to_s}.yaml",'w')
    f.write data.to_yaml
    f.close
  end
  
  def write_txt(id,data)
    f = File.open(backup_dir+"/#{id.to_s}.txt",'w')
    f.write data.to_yaml
    f.close
  end
 
  def backup_dir
    File.expand_path("~/.twitter_backup/tweets")
  end
  
  def mkdir
    `mkdir -p #{backup_dir}`
  end
end