joshwand/tumblr-likes-downloader.rb

## tumblr-likes-downloader.rb
require 'tumblr_client'
require 'mechanize'
require 'date'
require 'yaml'
require 'uri'

Tumblr.configure do |config|
  config.consumer_key = "consumer_key"
  config.consumer_secret = "consumer_secret"
  config.oauth_token = "oath_token"
  config.oauth_token_secret = "token_secret"
end

THREADPOOL_SIZE = 4
SEGMENT_SIZE = 25

HISTORY_FILE = File.join(File.dirname(__FILE__), "previous.yml")


previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []

directory = "tumblr-likes"

client = Tumblr::Client.new
likes = client.likes
liked_count = likes["liked_count"]

puts liked_count

likes = []

(0...liked_count).each do |i|
  if i==0 or i%SEGMENT_SIZE==0
    p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
    client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
     likes << like if like['type'] == 'photo' and !previous_dled_ids.include?(like['id'])
   end
 end

end

if likes.empty?
  p "no new likes!"
  exit 0
end

puts "#{likes.count} new likes!"


# some of this code comes from https://github.com/jamiew/tumblr-photo-downloader

already_had = 0

threads = []
likes.each_slice(likes.count / THREADPOOL_SIZE ).each do |group|

  threads << Thread.new {
    begin
      p "launching thread #{threads.size + 1}"

      group.each do |like|

        i = 0
        like["photos"].each do |photo|
          url = photo["original_size"]["url"]

          filename = "#{like["blog_name"]}-#{like["slug"]}-"
          filename += "#{i}-" if i > 0
          filename += File.basename(URI.parse(url).path.split('?')[0])
          if File.exists?("#{directory}/#{filename}")
            puts "Already have #{url}"
            already_had += 1
          else
            begin
              puts "Saving photo #{url}"
              file = Mechanize.new.get(url)
              file.save_as("#{directory}/#{filename}")
            rescue Mechanize::ResponseCodeError => e
              puts "Error #{e.response_code} getting file for #{url}"
            end
          end
          i += 1
          previous_dled_ids << like['id']
        end
      end
    rescue Exception => e
      puts "unhandled exception:, #{$!}"
    end
    p "closing thread"
  }

end

threads.each{|t| t.join }

YAML.dump(previous_dled_ids, File.open(HISTORY_FILE, "w"))
	require 'tumblr_client'
	require 'mechanize'
	require 'date'
	require 'yaml'
	require 'uri'

	Tumblr.configure do \|config\|
	config.consumer_key = "consumer_key"
	config.consumer_secret = "consumer_secret"
	config.oauth_token = "oath_token"
	config.oauth_token_secret = "token_secret"
	end

	THREADPOOL_SIZE = 4
	SEGMENT_SIZE = 25

	HISTORY_FILE = File.join(File.dirname(__FILE__), "previous.yml")


	previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []

	directory = "tumblr-likes"

	client = Tumblr::Client.new
	likes = client.likes
	liked_count = likes["liked_count"]

	puts liked_count

	likes = []

	(0...liked_count).each do \|i\|
	if i==0 or i%SEGMENT_SIZE==0
	p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
	client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do \|like\|
	likes << like if like['type'] == 'photo' and !previous_dled_ids.include?(like['id'])
	end
	end

	end

	if likes.empty?
	p "no new likes!"
	exit 0
	end

	puts "#{likes.count} new likes!"



	# some of this code comes from https://github.com/jamiew/tumblr-photo-downloader

	already_had = 0

	threads = []
	likes.each_slice(likes.count / THREADPOOL_SIZE ).each do \|group\|

	threads << Thread.new {
	begin
	p "launching thread #{threads.size + 1}"

	group.each do \|like\|

	i = 0
	like["photos"].each do \|photo\|
	url = photo["original_size"]["url"]

	filename = "#{like["blog_name"]}-#{like["slug"]}-"
	filename += "#{i}-" if i > 0
	filename += File.basename(URI.parse(url).path.split('?')[0])
	if File.exists?("#{directory}/#{filename}")
	puts "Already have #{url}"
	already_had += 1
	else
	begin
	puts "Saving photo #{url}"
	file = Mechanize.new.get(url)
	file.save_as("#{directory}/#{filename}")
	rescue Mechanize::ResponseCodeError => e
	puts "Error #{e.response_code} getting file for #{url}"
	end
	end
	i += 1
	previous_dled_ids << like['id']
	end
	end
	rescue Exception => e
	puts "unhandled exception:, #{$!}"
	end
	p "closing thread"
	}

	end

	threads.each{\|t\| t.join }

	YAML.dump(previous_dled_ids, File.open(HISTORY_FILE, "w"))