Skip to content

Instantly share code, notes, and snippets.

@stefanoc
Last active September 24, 2015 12:28
Show Gist options
  • Save stefanoc/748352 to your computer and use it in GitHub Desktop.
Save stefanoc/748352 to your computer and use it in GitHub Desktop.
A simple script to export your FF data, based on http://bit.ly/fQ8Ocv
require "json"
require "open-uri"
require "optparse"
require "fileutils"
require "cgi"
class FriendfeedExporter
attr_reader :all_entries
def initialize(options)
@username = options[:username]
@remote_key = options[:remote_key]
@media_dir = "#{options[:media_dir]}/#{@username}"
@download_images = options[:download_images]
@download_files = options[:download_files]
@items_per_page = options[:items_per_page] || 100
@max_pages = options[:max_pages]
@credentials = "#{@username}:#{@remote_key}@" if @remote_key
@base_url = "https://#{@credentials}friendfeed-api.com/v2/feed/#{@username}?pretty=1&num=#{@items_per_page}&maxcomments=10000&maxlikes=10000&raw=1&"
@all_entries = []
FileUtils.mkdir_p(@media_dir) if @download_images || @download_files
end
def fetch_data(start)
url = @base_url + "start=#{start}"
data = JSON.load(open(url))
data["entries"].each do |entry|
download_images_for(entry) if @download_images
download_files_for(entry) if @download_files
end
@all_entries.concat(data["entries"])
data["entries"].size
end
def run
start = 0
loop do
fetched_entries = fetch_data(start)
puts "Fetched #{fetched_entries} items"
start += @items_per_page
break if @max_pages && start / @items_per_page >= @max_pages
break if fetched_entries < @items_per_page
end
end
def download_images_for(entry)
entry["thumbnails"].select { |tn| tn["url"].index("m.friendfeed-media.com") }.each_with_index do |tn, i|
save_image(tn["link"], entry["id"].gsub("e/", "i_") + ".#{i}")
save_image(tn["url"], entry["id"].gsub("e/", "t_") + ".#{i}")
end if entry.has_key?("thumbnails") && !entry["thumbnails"].empty?
end
def download_files_for(entry)
entry["files"].select { |tn| tn["url"].index("m.friendfeed-media.com") }.each_with_index do |tn, i|
File.open(File.join(@media_dir, entry["id"].gsub("e/", "f_") + ".#{i}"), "w") do |of|
of.write(open(tn["url"]).read)
end
end if entry.has_key?("files") && !entry["files"].empty?
end
def save_image(url, base_name)
url = url.gsub("http://", "http://#{@credentials}") if @credentials
open(url) do |f|
output_file = File.join(@media_dir, base_name) + case f.content_type
when "image/jpeg" then ".jpg"
when "image/png" then ".png"
when "image/gif" then ".gif"
else ".unknown"
end
File.open(output_file, "w") { |of| of.write(f.read) }
end
end
end
options = {:media_dir => "./ff-media", :download_images => false, :download_files => false, :items_per_page => 100, :max_pages => nil}
OptionParser.new do |opts|
opts.on("--username USERNAME") { |v| options[:username] = v }
opts.on("--media-dir DIR") { |v| options[:media_dir] = v }
opts.on("--remote-key KEY") { |v| options[:remote_key] = v }
opts.on("--[no-]download-images") { |v| options[:download_images] = v }
opts.on("--[no-]download-files") { |v| options[:download_files] = v }
opts.on("--items-per-page ITEMS") { |v| options[:items_per_page] = v.to_i }
opts.on("--max-pages PAGES") { |v| options[:max_pages] = v.to_i }
end.parse!
exp = FriendfeedExporter.new(options)
exp.run
puts exp.all_entries.to_json
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment