Skip to content

Instantly share code, notes, and snippets.

@ConnorWGarvey
Last active April 26, 2018 22:03
Show Gist options
  • Save ConnorWGarvey/5a868543705eac6e3c747c9b875892f9 to your computer and use it in GitHub Desktop.
Save ConnorWGarvey/5a868543705eac6e3c747c9b875892f9 to your computer and use it in GitHub Desktop.
Flickr Downloader
#!/usr/bin/env ruby
require 'fileutils'
require 'flickraw'
require 'open-uri'
require 'set'
require 'zaru'
def download_page(destination_dir:, page:)
count = 0
flickr.photos.search(page:page, per_page:300, sort:'date-posted-asc', user_id:'me').each do |p|
succeeded = false
while !succeeded
begin
download_photo destination_dir:destination_dir, p:p
succeeded = true
rescue Net::OpenTimeout, JSON::ParserError, Net::ReadTimeout
puts 'Error getting the next list of photos to download... waiting to try again...'
sleep 20
end
end
count += 1
end
count
end
def download_photo(p:, destination_dir:)
existing_file = downloaded(directory:destination_dir, id:p.id)
if existing_file && !downsized?(existing_file)
puts "#{p.id} exists. skipping."
return
end
info = flickr.photos.getInfo(photo_id:p.id)
url = FlickRaw.url_o(info)
basic_name = "#{p.id};#{info.dates.taken.gsub(':', '-')},#{info.title}"
extension = url[url.rindex('.')+1..-1]
name = Zaru.sanitize! basic_name, padding:4
destination = "#{destination_dir}/#{name}.#{extension}"
puts name
if File.exist?(destination)
puts ' - exists. skipping.'
return
end
original = true
succeeded = false
sizes = ['_n', '', '_z', '_b'] # n=320, ''=500, z=640, b=1024
while !succeeded
begin
IO.copy_stream(open(url), destination)
if FileUtils.identical?(destination, 'missing-file.jpg')
File.delete(destination)
destination = "#{destination_dir}/#{name} (NOT ORIGINAL).#{extension}"
puts " - missing. downloading smaller size to #{destination}."
url = FlickRaw.send("url#{sizes.pop}", info)
original = false
else
File.delete(existing_file) if existing_file && (original == true)
succeeded = true
end
rescue Net::OpenTimeout, Net::ReadTimeout
puts ' - error downloading... waiting to try again...'
sleep 20
end
end
end
def downloaded(directory:, id:)
Dir["#{directory}/*"].each do |f|
return f if File.basename(f).start_with?("#{id};")
end
return nil
end
def downsized?(f)
File.basename(f).include?('(NOT ORIGINAL)')
end
FlickRaw.api_key = '...your API key...'
FlickRaw.shared_secret = '...your shared secret...'
token = flickr.get_request_token
auth_url = flickr.get_authorize_url(token['oauth_token'], :perms => 'delete')
puts "Open to complete authication: #{auth_url}"
puts 'Paste the generated token here.'
verify = gets.strip
begin
flickr.get_access_token(token['oauth_token'], token['oauth_token_secret'], verify)
login = flickr.test.login
username = login.username
# flickr.access_token = '...your access token if you don't want to reauthenticate...'
# flickr.access_secret = '...your secret if you don't want to reauthenticate...'
# username = '...your username so that it can be printed below...'
puts "You are now authenticated as #{username} with token #{flickr.access_token} and secret #{flickr.access_secret}"
destination_dir = 'downloaded'
destination_dir = 'download'
FileUtils.mkdir_p destination_dir
# existing = Set.new(
# Dir["#{destination_dir}/*"].map do |f|
# /^\d+/.match(File.basename(f))[0].to_i
# end
# )
page = 1
loop do
count = 0
succeeded = false
while !succeeded
begin
count = download_page destination_dir:destination_dir, page:page
succeeded = true
rescue Net::OpenTimeout, JSON::ParserError, Net::ReadTimeout
puts 'Timed out... waiting...'
sleep 20
end
end
page += 1
break if count == 0
end
rescue FlickRaw::FailedResponse => e
puts "Authentication failed : #{e.msg}"
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment