Last active
September 24, 2015 12:28
-
-
Save stefanoc/748352 to your computer and use it in GitHub Desktop.
A simple script to export your FF data, based on http://bit.ly/fQ8Ocv
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "json" | |
require "open-uri" | |
require "optparse" | |
require "fileutils" | |
require "cgi" | |
class FriendfeedExporter | |
attr_reader :all_entries | |
def initialize(options) | |
@username = options[:username] | |
@remote_key = options[:remote_key] | |
@media_dir = "#{options[:media_dir]}/#{@username}" | |
@download_images = options[:download_images] | |
@download_files = options[:download_files] | |
@items_per_page = options[:items_per_page] || 100 | |
@max_pages = options[:max_pages] | |
@credentials = "#{@username}:#{@remote_key}@" if @remote_key | |
@base_url = "https://#{@credentials}friendfeed-api.com/v2/feed/#{@username}?pretty=1&num=#{@items_per_page}&maxcomments=10000&maxlikes=10000&raw=1&" | |
@all_entries = [] | |
FileUtils.mkdir_p(@media_dir) if @download_images || @download_files | |
end | |
def fetch_data(start) | |
url = @base_url + "start=#{start}" | |
data = JSON.load(open(url)) | |
data["entries"].each do |entry| | |
download_images_for(entry) if @download_images | |
download_files_for(entry) if @download_files | |
end | |
@all_entries.concat(data["entries"]) | |
data["entries"].size | |
end | |
def run | |
start = 0 | |
loop do | |
fetched_entries = fetch_data(start) | |
puts "Fetched #{fetched_entries} items" | |
start += @items_per_page | |
break if @max_pages && start / @items_per_page >= @max_pages | |
break if fetched_entries < @items_per_page | |
end | |
end | |
def download_images_for(entry) | |
entry["thumbnails"].select { |tn| tn["url"].index("m.friendfeed-media.com") }.each_with_index do |tn, i| | |
save_image(tn["link"], entry["id"].gsub("e/", "i_") + ".#{i}") | |
save_image(tn["url"], entry["id"].gsub("e/", "t_") + ".#{i}") | |
end if entry.has_key?("thumbnails") && !entry["thumbnails"].empty? | |
end | |
def download_files_for(entry) | |
entry["files"].select { |tn| tn["url"].index("m.friendfeed-media.com") }.each_with_index do |tn, i| | |
File.open(File.join(@media_dir, entry["id"].gsub("e/", "f_") + ".#{i}"), "w") do |of| | |
of.write(open(tn["url"]).read) | |
end | |
end if entry.has_key?("files") && !entry["files"].empty? | |
end | |
def save_image(url, base_name) | |
url = url.gsub("http://", "http://#{@credentials}") if @credentials | |
open(url) do |f| | |
output_file = File.join(@media_dir, base_name) + case f.content_type | |
when "image/jpeg" then ".jpg" | |
when "image/png" then ".png" | |
when "image/gif" then ".gif" | |
else ".unknown" | |
end | |
File.open(output_file, "w") { |of| of.write(f.read) } | |
end | |
end | |
end | |
options = {:media_dir => "./ff-media", :download_images => false, :download_files => false, :items_per_page => 100, :max_pages => nil} | |
OptionParser.new do |opts| | |
opts.on("--username USERNAME") { |v| options[:username] = v } | |
opts.on("--media-dir DIR") { |v| options[:media_dir] = v } | |
opts.on("--remote-key KEY") { |v| options[:remote_key] = v } | |
opts.on("--[no-]download-images") { |v| options[:download_images] = v } | |
opts.on("--[no-]download-files") { |v| options[:download_files] = v } | |
opts.on("--items-per-page ITEMS") { |v| options[:items_per_page] = v.to_i } | |
opts.on("--max-pages PAGES") { |v| options[:max_pages] = v.to_i } | |
end.parse! | |
exp = FriendfeedExporter.new(options) | |
exp.run | |
puts exp.all_entries.to_json |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment