Import and update your Pinboard bookmarks to DEVONthink
(* Import and update your Pinboard bookmarks to DEVONthink | |
Based on work done by Christian Grunenberg on Mon Jan 23 2006, | |
Rafael Bugajewski to support Pinboard instead of Delicious on Sun Dec 19 2010 and | |
Andreas Zeitler on Sun Mar 03 2011 to display user feedback when finished. | |
By Sascha A. Carlin <https://sascha.carlin.de/> on 2018-03-07 to set the creation date of new record, show progress bar, use Pinboard Auth Token, use modification date of folder to fetch only recent items | |
Copyright (c) 2018. All rights reserved. *) | |
use framework "Foundation" | |
use scripting additions | |
-- Set this to the file path of the DT database you want your Pinboard bookmarks in. Would usually look like ~/Documents/Database.dtBase2 | |
property dtDatabase : "" | |
-- We use your Pinboard Auth Token. See https://pinboard.in/settings/password | |
property pAuthToken : "" | |
-- We support different import strategies. See below for details. | |
property importStrategy : "decluttered" | |
-- 1…3. The higher, the more we send to DTs log | |
property logVerbosity : 1 | |
-- If true, sends all log info to AppleScript's logging system | |
property scriptDebugging : false | |
on run argv | |
-- Assume we should display dialogs | |
set silentMode to false | |
-- Check for command line argument silent. If present, omit all dialogs | |
try | |
if (item 1 of argv is "silent") then | |
set silentMode to true | |
end if | |
end try | |
my logMessage("silentMode", silentMode, 3) | |
-- Extract username from Auth token, we'll display it in the progress bar | |
set _words to words of pAuthToken | |
set pUser to item 1 of _words | |
-- We create the group in case it does not exist in the database | |
-- We check whether to import all bookmarks from Pinboard or just the ones younger than a certain date | |
set theUTCDateTime to "" | |
set logUpdateFrom to "" | |
set theGroupComment to "" | |
tell application id "DNtp" | |
set theDatabase to open database dtDatabase | |
set theGroup to get record at "/Pinboard" in theDatabase | |
if (theGroup is missing value or type of theGroup is not group) then | |
-- If the group does not exist, we'l pull all bookmarks from Pinboard | |
set theGroup to create location "/Pinboard" in theDatabase | |
set thumbnail of theGroup to "https://imgur.com/download/UfP65so" -- PNG of a macOS folder with embosses Pinboard pin. Created using Lucas Garron's Folderify: https://github.com/lgarron/folderify | |
set logUpdateFrom to "Group not found or has no comment, assuming we need to start from scratch!" | |
else | |
set theGroupComment to theGroup's comment as text | |
if theGroupComment is not "" then | |
-- We use the group's comment to store the datetime of the last Pinboard post to be imported. | |
-- Create date string as per RFC 3339 for Pinboards API | |
try | |
tell application "System Events" to set theDate to (my dateFromRfc3339String:theGroupComment) as date | |
set theUTCDateTime to (my rfc3339FromDate:theDate) | |
set logUpdateFrom to "Group has comment, we start from " & theUTCDateTime | |
on error error_message number error_number | |
my logMessage(error_message, error_number, 4) | |
set theUTCDateTime to "" | |
end try | |
else | |
set logUpdateFrom to "Group has no comment, assuming we need to start from scratch!" | |
end if | |
end if | |
end tell | |
logMessage("Import strategy:", importStrategy, 1) | |
logMessage("Update strategy:", logUpdateFrom, 1) | |
logMessage("Import posts created on or after:", theUTCDateTime, 1) | |
-- Set up the request parameters for Pinboard, telling it to deliver all or only entries younger than theUTCDateTime | |
if (theUTCDateTime is not "") then | |
set thePinboardURL to "https://api.pinboard.in/v1/posts/all?auth_token=" & pAuthToken & "&fromdt=" & theUTCDateTime | |
else | |
-- Debug Switch: Be nice to Pinboard and use recent while debugging. | |
if scriptDebugging then | |
set thePinboardURL to "https://api.pinboard.in/v1/posts/recent?auth_token=" & pAuthToken | |
else | |
set thePinboardURL to "https://api.pinboard.in/v1/posts/all?auth_token=" & pAuthToken | |
end if | |
end if | |
logMessage("thePinboardURL:", thePinboardURL, 2) | |
-- Get the data and process the entries | |
tell application id "DNtp" | |
try | |
set theXML to download markup from thePinboardURL encoding "UTF-8" | |
if (theXML is missing value or theXML is "") then | |
error "Download failed." | |
else if theXML contains "503 Service Temporarily Unavailable" then | |
error "503 Service Temporarily Unavailable" | |
logMessage("theXML:", theXML, 3) | |
else | |
tell application "System Events" | |
set x to make new XML data with data theXML | |
set theElements to XML elements of (XML element 1 of x) -- <posts> | |
end tell | |
set theElementCount to length of theElements | |
my logMessage("Elements to process from XML:", theElementCount as string, 1) | |
if (theElementCount < 1) then | |
error "Pinboard says it found no new bookmarks." | |
end if | |
try | |
tell application id "DNtp" | |
show progress indicator "Importing " & theElementCount & " bookmarks from Pinboard (" & pUser & ") …" steps (theElementCount) with cancel button | |
end tell | |
set importedPosts to 0 | |
-- Import loop | |
repeat with theSteps from theElementCount to 1 by -1 | |
-- Get URL first. We check whether it's unique. | |
tell application "System Events" | |
set theElement to item theSteps of theElements | |
set theUrl to (value of XML attribute named "href" of theElement) as string | |
end tell | |
tell application id "DNtp" | |
if not (exists record with URL theUrl) then | |
-- URL is unique, let's go! | |
-- Show what we're working on | |
step progress indicator theUrl | |
my logMessage(theUrl, "…", 3) | |
-- Extract data from XML | |
tell application "System Events" | |
set theName to (value of XML attribute named "description" of theElement) as string | |
set theTag to (value of XML attribute named "tag" of theElement) as string | |
set theComment to (value of XML attribute named "extended" of theElement) as string | |
set theDate to (value of XML attribute named "time" of theElement) as string | |
end tell | |
tell application "System Events" | |
-- Get pathExtension and convert to lowercase. We'll use it to handle binary files. | |
-- URLs containing UTF characters seem to cause NSURL to barf :-( | |
set theExtension to "" | |
try | |
set _url to (current application's |NSURL|'s URLWithString:theUrl) | |
set theExtension to _url's pathExtension as string | |
tell me to set the theExtension to do shell script "echo " & quoted form of (theExtension) & " | tr A-Z a-z" | |
end try | |
set {od, AppleScript's text item delimiters} to {AppleScript's text item delimiters, " "} | |
set theTags to {} | |
try | |
set theTags to (text items of theTag) | |
end try | |
set AppleScript's text item delimiters to od | |
end tell | |
try | |
-- Judging by theExtension, some files should be processed as they come. | |
-- We differentiate between "doucment" and binary formats. | |
if (theExtension is in {"pdf", "jpg", "jpeg", "png", "zip", "rtf", "csv", "dmg", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "key", "pages", "key", "numbers"}) then | |
set theRecord to create web document from theUrl in theGroup | |
else if (theExtension is in {"zip", "dmg", "exe", "tar", "gz"}) then | |
set theRecord to create record with {type:bookmark, rich text:theComment} in theGroup | |
else | |
(* Choose an IMPORT STRATEGY. | |
See also DEVONthink's blog: http://blog.devontechnologies.com/2010/05/tuesday-tip-capturing-web-content/ *) | |
if (importStrategy is "create web document") then | |
(* Slow. As good as opening the URL in your favorite browser, includes all media & scripts. | |
Great web-like format, until Apple breaks backwards compatability. *) | |
-- set theRecord to create web document from "http://instapaper.com/text?u=" & (my encodeText:theUrl) in theGroup | |
set theRecord to create web document from theUrl in theGroup | |
else if (importStrategy is "create PDF document") then | |
(* Slow. PDF version of the site's print layout. | |
PDFs preserve most of the original style, most links can still be used. | |
Probably the best for serious archiving. | |
Todo: How does DEVONthink determine page width? *) | |
set theRecord to create PDF document from theUrl in theGroup with pagination | |
else if (importStrategy is "create formatted note") then | |
(* Slow. If your really can do without the original formatting and remote images. | |
Really, I don't see the point. Can as well use Markdown and save some execution time. *) | |
set theRecord to create formatted note from theUrl in theGroup | |
else if (importStrategy is "decluttered") then | |
(* Great for properly marked up HTML. Pretty bad for everthing else. | |
Want to use the same declutter functionality as in the DEVONthink Web Clipper? | |
Uses an external service! *) | |
-- Alternativly you can use Instapaper. | |
-- See the comment above at "create web document from" | |
set theMarkDownUrl to "http://heckyesmarkdown.com/go/?read=1&md=1&u=" & (my encodeText:theUrl) | |
set theContent to download markup from theMarkDownUrl | |
-- If the Markdown content seems to be too short, grab a PDF instead. | |
if (length of theContent < 400) then | |
set theRecord to create PDF document from theUrl in theGroup with pagination | |
else | |
set theRecord to create record with {type:markdown, content:theContent} in theGroup | |
end if | |
else if (importStrategy is "create Markdown") then | |
(* Rather not use an external service? *) | |
set theRecord to create Markdown from theUrl in theGroup | |
else if (importStrategy is "create record") then | |
(* Fastest method. Doesn't store anything local. | |
If you do not need any content but just the URLs, then choose this. *) | |
set theRecord to create record with {type:bookmark, rich text:theComment} in theGroup | |
end if | |
end if | |
-- Populate the record's fields with your Pinboard data | |
set the name of theRecord to theName | |
set the tags of theRecord to theTags | |
set the URL of theRecord to theUrl | |
set the comment of theRecord to theComment | |
-- Properly format theDate! For some functions, DEVONthink need an AppleScript date type as theDate | |
set the date of theRecord to (my dateFromRfc3339String:theDate) | |
-- Store the datetime of the last successfully imported post as a comment of the group. We'll use it to determine from which date onwards to ask for new posts the next time the script runs. | |
-- Doing so on every post ensures that we'll never miss a post in case the import process breaks. | |
try | |
set theGroup's comment to theDate | |
set importedPosts to importedPosts + 1 | |
end try | |
on error error_message number error_number | |
-- Something went wrong. | |
my logMessage(theUrl, error_message, 0) | |
end try | |
else | |
my logMessage(theUrl, "Skipped as duplicate", 3) | |
end if | |
-- I like percentages in my logs. | |
set thePercentage to theSteps / theElementCount | |
set thePercentage to round thePercentage * 100 rounding down | |
my logMessage(theUrl, (thePercentage as string) & "% of " & theElementCount, 4) -- I am lazy. Log level 4 is a magic number. When you set scriptDebugging to true, this info will be logged to AppleScript's log but will never show up in DT. | |
if cancelled progress then exit repeat | |
end tell | |
end repeat | |
on error error_message number error_number | |
hide progress indicator | |
if silentMode is false then | |
display alert "Pinboard importer" message error_message as warning | |
else | |
my logMessage(error_number, error_message, 0) | |
end if | |
end try | |
end if | |
-- When done, play a sound | |
hide progress indicator | |
tell application "Finder" | |
if exists POSIX file "/System/Library/Sounds/Glass.aiff" then | |
do shell script "afplay /System/Library/Sounds/Glass.aiff" | |
end if | |
end tell | |
if silentMode is false then | |
display alert "Done importing." & return & return & "Processed bookmarks: " & theElementCount & return & "Skipped:" & (theElementCount - importedPosts) & return & "Imported: " & importedPosts | |
end if | |
on error error_message number error_number | |
hide progress indicator | |
if silentMode is false and the error_number is not -128 then display alert "Pinboard importer" message error_message as warning | |
my logMessage(error_number, error_message, 4) | |
end try | |
end tell | |
end run | |
-- Stolen from Mark Aldritt: http://forum.latenightsw.com/t/formatting-dates/841 | |
on rfc3339FromDate:aDate | |
set theFormatter to current application's NSDateFormatter's new() | |
theFormatter's setLocale:(current application's NSLocale's localeWithLocaleIdentifier:"en_US_POSIX") | |
theFormatter's setTimeZone:(current application's NSTimeZone's timeZoneWithAbbreviation:"GMT") -- skip for local time | |
theFormatter's setDateFormat:"yyyy'-'MM'-'dd'T'HH':'mm':'ssXXX" | |
return (theFormatter's stringFromDate:aDate) as text | |
end rfc3339FromDate: | |
-- Stolen from Mark Aldritt: http://forum.latenightsw.com/t/formatting-dates/841 | |
on dateFromRfc3339String:theString | |
set theFormatter to current application's NSDateFormatter's new() | |
theFormatter's setLocale:(current application's NSLocale's localeWithLocaleIdentifier:"en_US_POSIX") | |
theFormatter's setTimeZone:(current application's NSTimeZone's timeZoneWithAbbreviation:"GMT") -- skip for local time | |
theFormatter's setDateFormat:"yyyy'-'MM'-'dd'T'HH':'mm':'ssXXX" | |
return (theFormatter's dateFromString:theString) as date | |
end dateFromRfc3339String: | |
-- Stolen from Apple: https://developer.apple.com/library/content/documentation/LanguagesUtilities/Conceptual/MacAutomationScriptingGuide/EncodeandDecodeText.html | |
on encodeText:theText | |
set theString to stringWithString_(theText) of NSString of current application | |
set theEncoding to NSUTF8StringEncoding of current application | |
set theAdjustedString to stringByAddingPercentEscapesUsingEncoding_(theEncoding) of theString | |
return (theAdjustedString as string) | |
end encodeText: | |
on logMessage(logResource, logInfo, logLevel) | |
if ((logLevel ≥ logVerbosity) and (logLevel < 4)) then | |
tell application id "DNtp" | |
log message logResource info logInfo | |
end tell | |
end if | |
if (scriptDebugging) then | |
log logInfo & ": " & logResource | |
end if | |
end logMessage |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment