Pull bookmarks from Kippt API, upload via Pinboard API

kippt_to_pinboard.rb

# Instructions

# install required packages by running:
# gem install json typhoeus awesome_print

# Download this .rb into a new directory on your PC (called something like "pinboard" or whatever)

# Fill out KIPPT_USERNAME, KIPPT_API_TOKEN, and PINBOARD_AUTH_TOKEN (below) by changing xxx to the real value.

# Run the script by running
# ruby kippt_to_pinboard.rb

# Wait a while, because it's 3 seconds per bookmark.

# Note that it will also generate some JSON backups of all your Kippt bookmarks inside the directory you're in.

KIPPT_USERNAME = "xxx"
KIPPT_API_TOKEN = "xxx" # This is somewhere on Kippt's website. I can't login so not sure where.
PINBOARD_AUTH_TOKEN = "xxx" # Get this from https://pinboard.in/settings/password. It looks like "username:123abc123abc123abc12"






require "rubygems"
require "json"
require "typhoeus"
require "awesome_print"
require "time"

# curl -X GET -H "X-Kippt-Username: xxx" -H "X-Kippt-API-Token: xxx" https://kippt.com/api/clips/

BASE_URL = "https://kippt.com"
API_URL = BASE_URL + "/api/"
MAX_RETRIES = 100

def kippt_request(url, with_api=false)
  if with_api
    full_url = BASE_URL + url
  else
    full_url = API_URL + url + "?limit=200"
  end

  if File.file?(filename = canonical_filename(full_url))
    puts "\tKippt API request recovered from file: " + filename
    body = File.open(filename, "rb").read
    
    fake_response = Struct.new(:effective_url, :body).new(full_url, body)
    fake_response.define_singleton_method("success?") {true}
    fake_request = Struct.new(:run).new(fake_response)
    return fake_request
  end
  
  puts "\tKippt API request: " + full_url
  request = Typhoeus::Request.new(
    full_url,
    :method => :get,
    :headers => {"X-Kippt-Username" => KIPPT_USERNAME, "X-Kippt-API-Token" => KIPPT_API_TOKEN},
    )
  return request
end

def hash_with_retries(request)
  retries = 0
  loop do
    return nil if retries >= MAX_RETRIES
    puts "\t\tRetry #{retries}" if retries > 0
    response = request.run
    if response.success?
      write_file(response)
      return JSON.parse(response.body)
    else
      retries += 1
    end
  end
end

def canonical_filename(url)
  url.gsub(API_URL, "").gsub("/", "_").gsub("&", "?")
end

def write_file(response)
  return unless response.success?
  filename = canonical_filename(response.effective_url)
  puts "\tWriting response to file: " + filename
  File.open(filename, 'w') { |file| file.write(response.body) }
end



bookmarks = []
all_tags = []



puts "Getting all lists."
lists = hash_with_retries(kippt_request("lists"))


puts "Found #{lists['objects'].size} lists!"

lists["objects"].each do |list_hash|
  list_title = list_hash["title"]
  id = list_hash["id"]

  puts
  puts "============================================="
  puts
  
  puts "Getting clips from list #{id}, #{list_title}"
  list = hash_with_retries(kippt_request("lists/" + id.to_s + "/clips"))

  puts "Found #{list['objects'].size} clips in this list!"


  list["objects"].each do |clip|
    hash_tags = []
    [clip["title"], clip["notes"]].each do |string|
      string = string.to_s
      hash_tags.push(*string.scan(/#\w+/).map {|tag| tag.gsub("#", "").downcase })
      hash_tags.push(*string.scan(/\W\wv\w\W/).map {|tag| tag.gsub(/\W/, "").downcase })
    end
    hash_tags = hash_tags.uniq.compact

    created = Time.at(clip["created"]).utc
    created_string = created.iso8601
    bookmarks << {
      :list_title => list_title,
      :list_tag => list_title.gsub(/\s+/, "-").downcase,
      :hash_tags => hash_tags,
      :url => clip["url"],
      :is_favorite => clip["is_favorite"],
      :title => clip["title"].to_s,
      :notes => clip["notes"].to_s,
      :created => created,
      :created_string => created_string,
    }
  end

  puts
  puts "============================================="
  puts "============================================="
  puts "============================================="
  puts

  # 2010-12-11T19:48:02Z
  
  # if next_url = list["meta"]["next"]
  #   puts "!!!! THERES A NEXT URL!!!  " + next_url
  # end

  # loop do
  #   if next_url = list["meta"]["next"]
  #     next_page = hash_with_retries(kippt_request(next_url, true))
  #   end
  # end
end

# https://api.pinboard.in/v1/post?auth_token=user:8c091591a9b21fa24eda

count = 0
total = bookmarks.count

bookmarks.each do |b|
  tags = Array[b[:list_tag]] + b[:hash_tags]
  tags = tags.uniq.compact
  tags = tags.select {|t| t.length != 1 && t != t.to_i.to_s}
  all_tags.push(*tags)

  if b[:title].length > 255
    b[:notes] = "Full title:\n #{b[:title]} \n\n #{b[:notes]}"
    b[:title] = b[:title][0,255]
  end
  # puts
  # puts b[:url] + " ...at... " + b[:created_string]
  # puts "#{b[:title]} ... #{b[:notes]}"
  # puts tags.join(", ")

  params = {
    :url => b[:url],
    :description => b[:title],
    :extended => b[:notes],
    :tags => tags.join(","),
    :dt => b[:created_string],
    :auth_token => PINBOARD_AUTH_TOKEN,
  }

  ps = params.map {|k, v| k.to_s + "=" + CGI.escape(v.to_s)}.join("&")

  pinboard_url = "https://api.pinboard.in/v1/posts/add?#{ps}"

  response = Typhoeus::Request.get(pinboard_url)

  count += 1
  if !response.success?
    puts "!!! FAILED #{response.code}. #{response.body}. #{pinboard_url}"
  end
  
  if count < 25 || count % 10 == 0
    # puts pinboard_url
    puts "Success? #{response.success?}.\tCode: #{response.code}.\t Done #{count}\t\tof #{total}"
  end

  sleep 3.1 - response.total_time
end

puts "ALL TAGS:"
puts all_tags.uniq.compact.sort
puts "longest title:"
puts bookmarks.map {|b| b[:title].length}.max
puts "longest notes:"
puts bookmarks.map {|b| b[:notes].length}.max
puts "number of bookmarks:"
puts bookmarks.length