Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
A script for gathering new Pinboard links with a certain tag and generating Markdown/Jekyll posts when enough are collected.
#!/usr/bin/ruby
# WebExcursions, a script for gathering new Pinboard links with a certain tag
# and generating Markdown/Jekyll posts when enough are collected.
# Brett Terpstra 2013
#
# -f to force writing out current bookmarks to file regardless of count
%w[fileutils set net/https zlib rexml/document time base64 uri cgi stringio].each do |filename|
require filename
end
$conf = {}
$conf['debug'] = true
# Pinboard credentials
$conf['user'] = ''
$conf['password'] = ''
# Where to store the database
$conf['db_location'] = File.expand_path("./webexcursions")
# Tag to use for finding bloggable bookmarks
$conf['blog_tag'] = 'blogit'
# How many posts must be gathered before publishing
$conf['min_count'] = 5
# relative location of folder for creating drafts
$conf['drafts_folder'] = 'source/_drafts'
# template for post headers
$conf['post_template'] =<<ENDTEMPLATE
---
title: "Web Excursions for #{Time.now.strftime('%B %d, %Y')}"
layout: post
tags:
- bookmarks
categories:
- Bookmarks
post_class: bookmarks
comments: false
---
ENDTEMPLATE
class Net::HTTP
alias_method :old_initialize, :initialize
def initialize(*args)
old_initialize(*args)
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
end
class Utils
def debug_msg(message,sticky = true)
if $conf['debug']
$stderr.puts message
end
end
end
class Pinboard
attr_accessor :user, :pass, :existing_bookmarks, :new_bookmarks
def initialize
# Make storage directory if needed
FileUtils.mkdir_p($conf['db_location'],:mode => 0755) unless File.exists? $conf['db_location']
# load existing bookmarks database
@existing_bookmarks = self.read_bookmarks
end
# Store a Marshal dump of a hash
def store obj = @existing_bookmarks, file_name = $conf['db_location']+'/bookmarks.stash', options={:gzip => false }
marshal_dump = Marshal.dump(obj)
file = File.new(file_name,'w')
file = Zlib::GzipWriter.new(file) unless options[:gzip] == false
file.write marshal_dump
file.close
return obj
end
# Load the Marshal dump to a hash
def load file_name
begin
file = Zlib::GzipReader.open(file_name)
rescue Zlib::GzipFile::Error
file = File.open(file_name, 'r')
ensure
obj = Marshal.load file.read
file.close
return obj
end
end
# Set up credentials for Pinboard.in
def set_auth(user,pass)
@user = user
@pass = pass
end
def new_bookmarks
return self.unique_bookmarks
end
def existing_bookmarks
@existing_bookmarks
end
# retrieves the XML output from the Pinboard API
def get_xml(api_call)
xml = ''
http = Net::HTTP.new('api.pinboard.in', 443)
http.use_ssl = true
http.start do |http|
request = Net::HTTP::Get.new(api_call)
request.basic_auth @user,@pass
response = http.request(request)
response.value
xml = response.body
end
return REXML::Document.new(xml)
end
# converts Pinboard API output to an array of URLs
def bookmarks_to_array(doc)
bookmarks = []
doc.elements.each('posts/post') do |ele|
post = {}
ele.attributes.each {|key,val|
post[key] = val;
}
bookmarks.push(post)
end
return bookmarks
end
# compares bookmark array to existing bookmarks to find new urls
def unique_bookmarks
xml = self.get_xml("/v1/posts/recent?tag=#{$conf['blog_tag']}&count=100")
bookmarks = self.bookmarks_to_array(xml)
unless @existing_bookmarks.nil?
old_hrefs = @existing_bookmarks.map { |x| x['href'] }
bookmarks.reject! { |s| old_hrefs.include? s['href'] }
end
return bookmarks
end
# wrapper for load
def read_bookmarks
# if the file exists, read it
if File.exists? $conf['db_location']+'/bookmarks.stash'
return self.load $conf['db_location']+'/bookmarks.stash'
else # new database
return []
end
end
def read_current_excursion
# if the file exists, read it
if File.exists? $conf['db_location']+'/current.stash'
return self.load $conf['db_location']+'/current.stash'
else # new database
return []
end
end
end
util = Utils.new
pb = Pinboard.new
pb.set_auth($conf['user'], $conf['password'])
# retrieve recent bookmarks
new_bookmarks = pb.new_bookmarks
# load the current draft stash
current_excursion = pb.read_current_excursion
if new_bookmarks.count > 0 || current_excursion.count > 0
util.debug_msg("Found #{new_bookmarks.count} unindexed bookmarks.",false)
else
util.debug_msg("No new bookmarks. Exiting.",false)
exit
end
# merge new bookmarks into main database
existing_hrefs = current_excursion.map { |x| x['href'] }
new_bookmarks.each {|bookmark|
pb.existing_bookmarks.push(bookmark)
unless existing_hrefs.include? bookmark['href']
current_excursion.push(bookmark)
end
}
pb.store
# if there are 5 or more bookmarks, create a draft post and clear cache
if current_excursion.length >= $conf['min_count'].to_i || ARGV[0] == '-f'
output = $conf['post_template']
current_excursion.each_with_index {|bookmark, i|
output += "[#{bookmark['description']}](#{bookmark['href']})\n"
output += ": #{bookmark['extended'].gsub(/\n+/,' ')}\n\n"
}
File.open("#{$conf['drafts_folder']}/web-excursions-#{Time.now.strftime('%B-%d-%Y').downcase}.md",'w+') do |f|
f.puts output
end
current_excursion = []
FileUtils.mv($conf['db_location']+'/current.stash',$conf['db_location']+"/published-#{Time.now.strftime('%Y-%m-%d-%s')}.stash")
else # fewer than five bookmarks, list existing and dump stash
puts "There are currently #{current_excursion.count} bookmarks collected."
current_excursion.each_with_index {|bookmark, i|
puts "#{i.to_s}: #{bookmark['description']}"
}
pb.store(current_excursion, $conf['db_location']+'/current.stash', options={:gzip => false })
end
@rretsiem

This comment has been minimized.

Copy link

rretsiem commented Jan 27, 2013

I'm getting the following error when enough bookmarks are collected:

Found 5 unindexed bookmarks.
/Users/<username>/.rbenv/versions/1.9.3-p362/lib/ruby/1.9.1/fileutils.rb:1515:in `stat': No such file or directory -     /Users/<username>/octopress/webexcursions/current.stash (Errno::ENOENT)
    from /Users/<username>/.rbenv/versions/1.9.3-p362/lib/ruby/1.9.1/fileutils.rb:1515:in `block in fu_each_src_dest'
from /Users/<username>/.rbenv/versions/1.9.3-p362/lib/ruby/1.9.1/fileutils.rb:1531:in `fu_each_src_dest0'
from /Users/<username>/.rbenv/versions/1.9.3-p362/lib/ruby/1.9.1/fileutils.rb:1513:in `fu_each_src_dest'
from /Users/<username>/.rbenv/versions/1.9.3-p362/lib/ruby/1.9.1/fileutils.rb:508:in `mv'
from webexcursions.rb:194:in `<main>'

Any idea?

@tdhopper

This comment has been minimized.

Copy link

tdhopper commented Sep 7, 2013

It's not working for me either.

/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:506:in `rename': No such file or directory - /Users/tdhopper/Downloads/webexcursions/current.stash or /Users/tdhopper/Downloads/webexcursions/published-2013-09-07-1378560701.stash (Errno::ENOENT)
    from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:506:in `mv'
    from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:1402:in `fu_each_src_dest'
    from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:1418:in `fu_each_src_dest0'
    from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:1400:in `fu_each_src_dest'
    from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/fileutils.rb:495:in `mv'
    from webexcursions.rb:194
@gglanzani

This comment has been minimized.

Copy link

gglanzani commented Apr 1, 2014

I've fixed the error by changing line 194 from

    FileUtils.mv($conf['db_location']+'/current.stash',$conf['db_location']+"/published-#{Time.now.strftime('%Y-%m-%d-%s')}.stash")

to

  begin
      FileUtils.mv($conf['db_location']+'/current.stash',$conf['db_location']+"/published-#{Time.now.strftime('%Y-%m-%d-%s')}.stash")
  rescue
      puts []
  end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.