Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Import and update your Pinboard bookmarks to DEVONthink
(* Import and update your Pinboard bookmarks to DEVONthink
Based on work done by Christian Grunenberg on Mon Jan 23 2006,
Rafael Bugajewski to support Pinboard instead of Delicious on Sun Dec 19 2010 and
Andreas Zeitler on Sun Mar 03 2011 to display user feedback when finished.
By Sascha A. Carlin <https://sascha.carlin.de/> on 2018-03-07 to set the creation date of new record, show progress bar, use Pinboard Auth Token, use modification date of folder to fetch only recent items
Copyright (c) 2018. All rights reserved. *)
use framework "Foundation"
use scripting additions
-- Set this to the file path of the DT database you want your Pinboard bookmarks in. Would usually look like ~/Documents/Database.dtBase2
property dtDatabase : ""
-- We use your Pinboard Auth Token. See https://pinboard.in/settings/password
property pAuthToken : ""
-- We support different import strategies. See below for details.
property importStrategy : "decluttered"
-- 1…3. The higher, the more we send to DTs log
property logVerbosity : 1
-- If true, sends all log info to AppleScript's logging system
property scriptDebugging : false
on run argv
-- Assume we should display dialogs
set silentMode to false
-- Check for command line argument silent. If present, omit all dialogs
try
if (item 1 of argv is "silent") then
set silentMode to true
end if
end try
my logMessage("silentMode", silentMode, 3)
-- Extract username from Auth token, we'll display it in the progress bar
set _words to words of pAuthToken
set pUser to item 1 of _words
-- We create the group in case it does not exist in the database
-- We check whether to import all bookmarks from Pinboard or just the ones younger than a certain date
set theUTCDateTime to ""
set logUpdateFrom to ""
set theGroupComment to ""
tell application id "DNtp"
set theDatabase to open database dtDatabase
set theGroup to get record at "/Pinboard" in theDatabase
if (theGroup is missing value or type of theGroup is not group) then
-- If the group does not exist, we'l pull all bookmarks from Pinboard
set theGroup to create location "/Pinboard" in theDatabase
set thumbnail of theGroup to "https://imgur.com/download/UfP65so" -- PNG of a macOS folder with embosses Pinboard pin. Created using Lucas Garron's Folderify: https://github.com/lgarron/folderify
set logUpdateFrom to "Group not found or has no comment, assuming we need to start from scratch!"
else
set theGroupComment to theGroup's comment as text
if theGroupComment is not "" then
-- We use the group's comment to store the datetime of the last Pinboard post to be imported.
-- Create date string as per RFC 3339 for Pinboards API
try
tell application "System Events" to set theDate to (my dateFromRfc3339String:theGroupComment) as date
set theUTCDateTime to (my rfc3339FromDate:theDate)
set logUpdateFrom to "Group has comment, we start from " & theUTCDateTime
on error error_message number error_number
my logMessage(error_message, error_number, 4)
set theUTCDateTime to ""
end try
else
set logUpdateFrom to "Group has no comment, assuming we need to start from scratch!"
end if
end if
end tell
logMessage("Import strategy:", importStrategy, 1)
logMessage("Update strategy:", logUpdateFrom, 1)
logMessage("Import posts created on or after:", theUTCDateTime, 1)
-- Set up the request parameters for Pinboard, telling it to deliver all or only entries younger than theUTCDateTime
if (theUTCDateTime is not "") then
set thePinboardURL to "https://api.pinboard.in/v1/posts/all?auth_token=" & pAuthToken & "&fromdt=" & theUTCDateTime
else
-- Debug Switch: Be nice to Pinboard and use recent while debugging.
if scriptDebugging then
set thePinboardURL to "https://api.pinboard.in/v1/posts/recent?auth_token=" & pAuthToken
else
set thePinboardURL to "https://api.pinboard.in/v1/posts/all?auth_token=" & pAuthToken
end if
end if
logMessage("thePinboardURL:", thePinboardURL, 2)
-- Get the data and process the entries
tell application id "DNtp"
try
set theXML to download markup from thePinboardURL encoding "UTF-8"
if (theXML is missing value or theXML is "") then
error "Download failed."
else if theXML contains "503 Service Temporarily Unavailable" then
error "503 Service Temporarily Unavailable"
logMessage("theXML:", theXML, 3)
else
tell application "System Events"
set x to make new XML data with data theXML
set theElements to XML elements of (XML element 1 of x) -- <posts>
end tell
set theElementCount to length of theElements
my logMessage("Elements to process from XML:", theElementCount as string, 1)
if (theElementCount < 1) then
error "Pinboard says it found no new bookmarks."
end if
try
tell application id "DNtp"
show progress indicator "Importing " & theElementCount & " bookmarks from Pinboard (" & pUser & ") …" steps (theElementCount) with cancel button
end tell
set importedPosts to 0
-- Import loop
repeat with theSteps from theElementCount to 1 by -1
-- Get URL first. We check whether it's unique.
tell application "System Events"
set theElement to item theSteps of theElements
set theUrl to (value of XML attribute named "href" of theElement) as string
end tell
tell application id "DNtp"
if not (exists record with URL theUrl) then
-- URL is unique, let's go!
-- Show what we're working on
step progress indicator theUrl
my logMessage(theUrl, "", 3)
-- Extract data from XML
tell application "System Events"
set theName to (value of XML attribute named "description" of theElement) as string
set theTag to (value of XML attribute named "tag" of theElement) as string
set theComment to (value of XML attribute named "extended" of theElement) as string
set theDate to (value of XML attribute named "time" of theElement) as string
end tell
tell application "System Events"
-- Get pathExtension and convert to lowercase. We'll use it to handle binary files.
-- URLs containing UTF characters seem to cause NSURL to barf :-(
set theExtension to ""
try
set _url to (current application's |NSURL|'s URLWithString:theUrl)
set theExtension to _url's pathExtension as string
tell me to set the theExtension to do shell script "echo " & quoted form of (theExtension) & " | tr A-Z a-z"
end try
set {od, AppleScript's text item delimiters} to {AppleScript's text item delimiters, " "}
set theTags to {}
try
set theTags to (text items of theTag)
end try
set AppleScript's text item delimiters to od
end tell
try
-- Judging by theExtension, some files should be processed as they come.
-- We differentiate between "doucment" and binary formats.
if (theExtension is in {"pdf", "jpg", "jpeg", "png", "zip", "rtf", "csv", "dmg", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "key", "pages", "key", "numbers"}) then
set theRecord to create web document from theUrl in theGroup
else if (theExtension is in {"zip", "dmg", "exe", "tar", "gz"}) then
set theRecord to create record with {type:bookmark, rich text:theComment} in theGroup
else
(* Choose an IMPORT STRATEGY.
See also DEVONthink's blog: http://blog.devontechnologies.com/2010/05/tuesday-tip-capturing-web-content/ *)
if (importStrategy is "create web document") then
(* Slow. As good as opening the URL in your favorite browser, includes all media & scripts.
Great web-like format, until Apple breaks backwards compatability. *)
-- set theRecord to create web document from "http://instapaper.com/text?u=" & (my encodeText:theUrl) in theGroup
set theRecord to create web document from theUrl in theGroup
else if (importStrategy is "create PDF document") then
(* Slow. PDF version of the site's print layout.
PDFs preserve most of the original style, most links can still be used.
Probably the best for serious archiving.
Todo: How does DEVONthink determine page width? *)
set theRecord to create PDF document from theUrl in theGroup with pagination
else if (importStrategy is "create formatted note") then
(* Slow. If your really can do without the original formatting and remote images.
Really, I don't see the point. Can as well use Markdown and save some execution time. *)
set theRecord to create formatted note from theUrl in theGroup
else if (importStrategy is "decluttered") then
(* Great for properly marked up HTML. Pretty bad for everthing else.
Want to use the same declutter functionality as in the DEVONthink Web Clipper?
Uses an external service! *)
-- Alternativly you can use Instapaper.
-- See the comment above at "create web document from"
set theMarkDownUrl to "http://heckyesmarkdown.com/go/?read=1&md=1&u=" & (my encodeText:theUrl)
set theContent to download markup from theMarkDownUrl
-- If the Markdown content seems to be too short, grab a PDF instead.
if (length of theContent < 400) then
set theRecord to create PDF document from theUrl in theGroup with pagination
else
set theRecord to create record with {type:markdown, content:theContent} in theGroup
end if
else if (importStrategy is "create Markdown") then
(* Rather not use an external service? *)
set theRecord to create Markdown from theUrl in theGroup
else if (importStrategy is "create record") then
(* Fastest method. Doesn't store anything local.
If you do not need any content but just the URLs, then choose this. *)
set theRecord to create record with {type:bookmark, rich text:theComment} in theGroup
end if
end if
-- Populate the record's fields with your Pinboard data
set the name of theRecord to theName
set the tags of theRecord to theTags
set the URL of theRecord to theUrl
set the comment of theRecord to theComment
-- Properly format theDate! For some functions, DEVONthink need an AppleScript date type as theDate
set the date of theRecord to (my dateFromRfc3339String:theDate)
-- Store the datetime of the last successfully imported post as a comment of the group. We'll use it to determine from which date onwards to ask for new posts the next time the script runs.
-- Doing so on every post ensures that we'll never miss a post in case the import process breaks.
try
set theGroup's comment to theDate
set importedPosts to importedPosts + 1
end try
on error error_message number error_number
-- Something went wrong.
my logMessage(theUrl, error_message, 0)
end try
else
my logMessage(theUrl, "Skipped as duplicate", 3)
end if
-- I like percentages in my logs.
set thePercentage to theSteps / theElementCount
set thePercentage to round thePercentage * 100 rounding down
my logMessage(theUrl, (thePercentage as string) & "% of " & theElementCount, 4) -- I am lazy. Log level 4 is a magic number. When you set scriptDebugging to true, this info will be logged to AppleScript's log but will never show up in DT.
if cancelled progress then exit repeat
end tell
end repeat
on error error_message number error_number
hide progress indicator
if silentMode is false then
display alert "Pinboard importer" message error_message as warning
else
my logMessage(error_number, error_message, 0)
end if
end try
end if
-- When done, play a sound
hide progress indicator
tell application "Finder"
if exists POSIX file "/System/Library/Sounds/Glass.aiff" then
do shell script "afplay /System/Library/Sounds/Glass.aiff"
end if
end tell
if silentMode is false then
display alert "Done importing." & return & return & "Processed bookmarks: " & theElementCount & return & "Skipped:" & (theElementCount - importedPosts) & return & "Imported: " & importedPosts
end if
on error error_message number error_number
hide progress indicator
if silentMode is false and the error_number is not -128 then display alert "Pinboard importer" message error_message as warning
my logMessage(error_number, error_message, 4)
end try
end tell
end run
-- Stolen from Mark Aldritt: http://forum.latenightsw.com/t/formatting-dates/841
on rfc3339FromDate:aDate
set theFormatter to current application's NSDateFormatter's new()
theFormatter's setLocale:(current application's NSLocale's localeWithLocaleIdentifier:"en_US_POSIX")
theFormatter's setTimeZone:(current application's NSTimeZone's timeZoneWithAbbreviation:"GMT") -- skip for local time
theFormatter's setDateFormat:"yyyy'-'MM'-'dd'T'HH':'mm':'ssXXX"
return (theFormatter's stringFromDate:aDate) as text
end rfc3339FromDate:
-- Stolen from Mark Aldritt: http://forum.latenightsw.com/t/formatting-dates/841
on dateFromRfc3339String:theString
set theFormatter to current application's NSDateFormatter's new()
theFormatter's setLocale:(current application's NSLocale's localeWithLocaleIdentifier:"en_US_POSIX")
theFormatter's setTimeZone:(current application's NSTimeZone's timeZoneWithAbbreviation:"GMT") -- skip for local time
theFormatter's setDateFormat:"yyyy'-'MM'-'dd'T'HH':'mm':'ssXXX"
return (theFormatter's dateFromString:theString) as date
end dateFromRfc3339String:
-- Stolen from Apple: https://developer.apple.com/library/content/documentation/LanguagesUtilities/Conceptual/MacAutomationScriptingGuide/EncodeandDecodeText.html
on encodeText:theText
set theString to stringWithString_(theText) of NSString of current application
set theEncoding to NSUTF8StringEncoding of current application
set theAdjustedString to stringByAddingPercentEscapesUsingEncoding_(theEncoding) of theString
return (theAdjustedString as string)
end encodeText:
on logMessage(logResource, logInfo, logLevel)
if ((logLevel logVerbosity) and (logLevel < 4)) then
tell application id "DNtp"
log message logResource info logInfo
end tell
end if
if (scriptDebugging) then
log logInfo & ": " & logResource
end if
end logMessage
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.