SearchLink creates Markdown links from automatic searches based on special syntax.

searchlink.rb

#!/usr/bin/env ruby
# encoding: utf-8

SILENT = ENV['SL_SILENT'] =~ /false/i ? false : true || true
VERSION = '2.2.23'

# SearchLink by Brett Terpstra 2015 <http://brettterpstra.com/projects/searchlink/>
# MIT License, please maintain attribution
require 'net/https'
require 'uri'
require 'rexml/document'
require 'shellwords'
require 'yaml'
require 'cgi'
require 'fileutils'
require 'zlib'
require 'time'
require 'json'
require 'erb'

if RUBY_VERSION.to_f > 1.9
  Encoding.default_external = Encoding::UTF_8
  Encoding.default_internal = Encoding::UTF_8
end

PINBOARD_CACHE = File.expand_path("~/.searchlink_cache")

class Array
  def longest_element
    group_by(&:size).max.last[0]
  end
end

class String
  def slugify
    downcase.gsub(/[^a-z0-9_]/i, '-').gsub(/-+/, '-')
  end

  def slugify!
    replace slugify
  end

  def clean
    gsub(/\n+/,' ').gsub(/"/,"&quot").gsub(/\|/,"-").gsub(/([&\?]utm_[scm].+=[^&\s!,\.\)\]]++?)+(&.*)/, '\2').sub(/\?&/,'').strip
  end

  def to_am # convert itunes to apple music link
    input = self.dup
    input.sub!(/\/itunes\.apple\.com/,'geo.itunes.apple.com')
    append = input =~ /\?[^\/]+=/ ? '&app=music' : '?app=music'
    input + append
  end

  def path_elements
    path = URI.parse(self).path
    path.sub!(/\/?$/,'/')
    path.sub!(/\/[^\/]+[\.\-][^\/]+\/$/,'')
    path.gsub!(/(^\/|\/$)/,'')
    path.split(/\//).delete_if {|section| section =~ /^\d+$/ || section.length < 5 }
  end

  def close_punctuation!
    self.replace close_punctuation
  end

  def close_punctuation
    return self unless self =~ /[“‘\[\(<]/

    words = self.split(/\s+/)

    punct_chars = {
      '“' => '”',
      '‘' => '’',
      '[' => ']',
      '(' => ')',
      '<' => '>'
    }
    left_punct = []

    words.each {|w|

      punct_chars.each {|k,v|

        if w =~ /#{Regexp.escape(k)}/
            left_punct.push(k)
        end

        if w =~ /#{Regexp.escape(v)}/
          left_punct.delete_at(left_punct.rindex(k))
        end
      }
    }
    tail = ""
    left_punct.reverse.each {|c|
      tail += punct_chars[c]
    }

    self.gsub(/[^a-z\)\]’”\.…]+$/i,"...").strip + tail

  end

  def remove_seo!(url)
    self.replace remove_seo(url)
  end

  def remove_seo(url)
    title = self.dup
    url = URI.parse(url)
    host = url.hostname
    path = url.path
    root_page = path =~ /^\/?$/ ? true : false

    title.gsub!(/\s*(&ndash;|&mdash;)\s*/,' - ')
    title.gsub!(/&[lr]dquo;/,'"')
    title.gsub!(/&[lr]dquo;/,"'")

    seo_title_separators = [ '|', '«', '-', '–', '·', ':' ]

    begin
      re_parts = []

      host_parts = host.sub(/(?:www\.)?(.*?)\.[^\.]+$/,'\1').split(/\./).delete_if {|p| p.length < 3 }
      h_re = host_parts.length > 0 ? host_parts.map{|seg| seg.downcase.split(//).join('.?') }.join('|') : ''
      re_parts.push(h_re) if h_re.length > 0

      # p_re = path.path_elements.map{|seg| seg.downcase.split(//).join('.?') }.join('|')
      # re_parts.push(p_re) if p_re.length > 0

      site_re = "(#{re_parts.join('|')})"

      dead_switch = 0

      while title.downcase.gsub(/[^a-z]/i,'') =~ /#{site_re}/i

        break if dead_switch > 5

        seo_title_separators.each_with_index do |sep, i|
          parts = title.split(/ ?#{Regexp.escape(sep)} +/)

          next if parts.length == 1
          remaining_separators = seo_title_separators[i..-1].map {|s| Regexp.escape(s)}.join('')
          seps = Regexp.new("^[^#{remaining_separators}]+$")

          longest = parts.longest_element.strip

          parts.delete_if {|pt|
            compressed = pt.strip.downcase.gsub(/[^a-z]/i,'')
            compressed =~ /#{site_re}/ && pt =~ seps ? !root_page : false
          } if parts.length > 1

          if parts.length == 0
            title = longest
          elsif parts.length < 2
            title = parts.join(sep)
          elsif parts.length > 2
            title = parts.longest_element.strip
          else
            title = parts.join(sep)
          end

        end
        dead_switch += 1
      end
    rescue Exception => e
      return self unless $cfg['debug']
      $stderr.puts "Error processing title"
      p e
      raise e
      # return self
    end

    seps = Regexp.new("[#{seo_title_separators.map {|s| Regexp.escape(s) }.join('')}]")
    if title =~ seps
      seo_parts = title.split(seps)
      if seo_parts.length > 1
        title = seo_parts.longest_element.strip
      end
    end

    title && title.length > 5 ? title.gsub(/\s+/,' ') : self
  end

  def truncate!(max)
    self.replace truncate(max)
  end

  def truncate(max)
    return self if self.length < max

    max -= 3
    counter = 0
    trunc_title = ""

    words = self.split(/\s+/)
    while trunc_title.length < max && counter < words.length
      trunc_title += " " + words[counter]
      break if trunc_title.length + 1 > max
      counter += 1
    end

    trunc_title = words[0] if trunc_title.nil? || trunc_title.length == 0

    trunc_title
  end

  def nil_if_missing
    if self =~ /missing value/
      return nil
    else
      self
    end
  end

  def split_hook
    elements = split(/\|\|/)
    {
      name: elements[0].nil_if_missing,
      url: elements[1].nil_if_missing,
      path: elements[2].nil_if_missing
    }
  end

  def split_hooks
    split(/\^\^/).map(&:split_hook)
  end
end

class SearchLink
  attr_reader :originput, :output, :clipboard
  attr_accessor :cfg
  # Values found in ~/.searchlink will override defaults in
  # this script

  def initialize(opt={})
    @printout = opt[:echo] || false
    unless File.exists? File.expand_path("~/.searchlink")
      default_config =<<ENDCONFIG
# set to true to have an HTML comment included detailing any errors
debug: true
# set to true to have an HTML comment included reporting results
report: true

# use Notification Center to display progress
notifications: false

# when running on a file, back up original to *.bak
backup: true

# change this to set a specific country for search (default US)
country_code: US

# set to true to force inline Markdown links
inline: false

# set to true to include a random string in reference titles.
# Avoids conflicts if you're only running on part of a document
# or using SearchLink multiple times within a document
prefix_random: true

# set to true to add titles to links based on the page title
# of the search result
include_titles: false

# confirm existence (200) of generated links. Can be disabled
# per search with `--v`, or enabled with `++v`.
validate_links: false

# If the link text is left empty, always insert the page title
# E.g. [](!g Search Text)
empty_uses_page_title: false

# Formatting for social links, use %service%, %user%, and %url%
# E.g. "%user% on %service%" => "ttscoff on Twitter"
#      "%service%/%user%" => "Twitter/ttscoff"
#      "%url%" => "twitter.com/ttscoff"
social_template: "%service%/%user%"

# append affiliate link info to iTunes urls, empty quotes for none
# example:
# itunes_affiliate: "&at=10l4tL&ct=searchlink"
itunes_affiliate: "&at=10l4tL&ct=searchlink"

# to create Amazon affiliate links, set amazon_partner to your amazon
# affiliate tag
#    amazon_partner: "bretttercom-20"
amazon_partner: "bretttercom-20"

# To create custom abbreviations for Google Site Searches,
# add to (or replace) the hash below.
# "abbreviation" => "site.url",
# This allows you, for example to use [search term](!bt)
# as a shortcut to search brettterpstra.com (using a site-specific
# Google search). Keys in this list can override existing
# search trigger abbreviations.
#
# If a custom search starts with "http" or "/", it becomes
# a simple replacement. Any instance of "$term" is replaced
# with a URL-escaped version of your search terms.
# Use $term1, $term2, etc. to replace in sequence from
# multiple search terms. No instances of "$term" functions
# as a simple shortcut. "$term" followed by a "d" lowercases
# the replacement. Use "$term1d," "$term2d" to downcase
# sequential replacements (affected individually).
# Long flags (e.g. --no-validate_links) can be used after
# any url in the custom searches.
#
# Use $terms to slugify all search terms, turning
# "Markdown Service Tools" into "markdown-service-tools"
custom_site_searches:
  bt: brettterpstra.com
  btt: https://brettterpstra.com/$term1d/$term2d
  bts: /search/$term --no-validate_links
  md: www.macdrifter.com
  ms: macstories.net
  dd: www.leancrew.com
  spark: macsparky.com
  man: http://man.cx/$term
  dev: developer.apple.com
  nq: http://nerdquery.com/?media_only=0&query=$term&search=1&category=-1&catid=&type=and&results=50&db=0&prefix=0
  gs: http://scholar.google.com/scholar?btnI&hl=en&q=$term&btnG=&as_sdt=80006
# Remove or comment (with #) history searches you don't want
# performed by `!h`. You can force-enable them per search, e.g.
# `!hsh` (Safari History only), `!hcb` (Chrome Bookmarks only),
# etc. Multiple types can be strung together: !hshcb (Safari
# History and Chrome bookmarks).
history_types:
- chrome_history
- chrome_bookmarks
- safari_bookmarks
- safari_history
# Pinboard search
# You can find your api key here: https://pinboard.in/settings/password
pinboard_api_key: ''

ENDCONFIG
      File.open(File.expand_path("~/.searchlink"), 'w') do |f|
        f.puts default_config
      end
    end

    @cfg = YAML.load_file(File.expand_path("~/.searchlink"))

    # set to true to have an HTML comment inserted showing any errors
    @cfg['debug'] ||= false

    # set to true to get a verbose report at the end of multi-line processing
    @cfg['report'] ||= false

    @cfg['backup'] = true unless @cfg.has_key? 'backup'

    # set to true to force inline links
    @cfg['inline'] ||= false

    # set to true to add titles to links based on site title
    @cfg['include_titles'] ||= false

    # set to true to use page title as link text when empty
    @cfg['empty_uses_page_title'] ||= false

    # change this to set a specific country for search (default US)
    @cfg['country_code'] ||= "US"

    # set to true to include a random string in ref titles
    # allows running SearchLink multiple times w/out conflicts
    @cfg['prefix_random'] = false unless @cfg['prefix_random']

    @cfg['social_template'] ||= '%service%/%user%'

    # append affiliate link info to iTunes urls, empty quotes for none
    # example:
    # $itunes_affiliate = "&at=10l4tL&ct=searchlink"
    @cfg['itunes_affiliate'] ||= '&at=10l4tL&ct=searchlink'

    # to create Amazon affiliate links, set amazon_partner to your amazon
    # affiliate tag
    #    amazon_partner: "bretttercom-20"
    @cfg['amazon_partner'] ||= ''

    # To create custom abbreviations for Google Site Searches,
    # add to (or replace) the hash below.
    # "abbreviation" => "site.url",
    # This allows you, for example to use [search term](!bt)
    # as a shortcut to search brettterpstra.com. Keys in this
    # hash can override existing search triggers.
    @cfg['custom_site_searches'] ||= {
      "bt" => "brettterpstra.com",
      "md" => "www.macdrifter.com"
    }

    # confirm existence of links generated from custom search replacements
    @cfg['validate_links'] ||= false

    # use notification center to show progress
    @cfg['notifications'] ||= false
    @cfg['pinboard_api_key'] ||= false

    @line_num = nil
    @match_column = nil
    @match_length = nil
  end

  def available_searches
    searches = [
      %w[a Amazon],
      %w[g Google],
      %w[wiki Wikipedia],
      ['s', 'Software search (Google)'],
      ['@t', 'Twitter user link'],
      ['@f', 'Facebook user link'],
      ['@l', 'LinkedIn user link'],
      ['@i', 'Instagram user link'],
      ['am', 'Apple Music'],
      ['amart', 'Apple Music Artist'],
      ['amalb', 'Apple Music Album'],
      ['amsong', 'Apple Music Song'],
      ['ampod', 'Apple Music Podcast'],
      ['ipod', 'iTunes podcast'],
      ['isong', 'iTunes song'],
      ['iart', 'iTunes artist'],
      ['ialb', 'iTunes album'],
      ['lsong', 'Last.fm song'],
      ['lart', 'Last.fm artist'],
      ['mas', 'Mac App Store'],
      ['masd', 'Mac App Store developer link'],
      ['itu', 'iTunes App Store'],
      ['itud', 'iTunes App Store developer link'],
      ['imov', 'iTunes Movies'],
      ['def', 'Dictionary definition'],
      ['hook', 'Hook Bookmarks'],
      %w[sp Spelling],
      %w[pb Pinboard]
      # ["h", "Web history"],
      # ["hs[hb]", "Safari [history, bookmarks]"],
      # ["hc[hb]", "Chrome [history, bookmarks]"]
    ]
    out = ''
    searches.each { |s| out += "!#{s[0]}#{spacer(s[0])}#{s[1]}\n" }
    out
  end

  def spacer(str)
    len = str.length
    str.scan(/[mwv]/).each { len += 1 }
    str.scan(/t/).each { len -= 1 }
    case len
    when 0..3
      "\t\t"
    when 4..12
      " \t"
    end
  end

  def help_text
    text = <<~EOHELP
      -- [Available searches] -------------------
      #{available_searches}
    EOHELP

    if @cfg['custom_site_searches']
      text += "\n-- [Custom Searches] ----------------------\n"
      @cfg['custom_site_searches'].each { |label, site| text += "!#{label}#{spacer(label)} #{site}\n" }
    end
    text
  end

  def help_dialog
    text = "[SearchLink v#{VERSION}]\n\n"
    text += help_text
    text += "\nClick \\\"More Help\\\" for additional information"
    res = `osascript <<'APPLESCRIPT'
set _res to display dialog "#{text.gsub(/\n/,'\\\n')}" buttons {"OK", "More help"} default button "OK" with title "SearchLink Help"

return button returned of _res
APPLESCRIPT
    `.strip
    `open http://brettterpstra.com/projects/searchlink` if res == 'More help'
  end

  def help_cli
    $stdout.puts help_text
  end

  def parse(input)
    @output = ''
    return false if input.empty?

    parse_arguments(input, { only_meta: true })
    @originput = input.dup

    if input.strip =~ /^help$/i
      if SILENT
        help_dialog # %x{open http://brettterpstra.com/projects/searchlink/}
      else
        $stdout.puts "SearchLink v#{VERSION}"
        $stdout.puts 'See http://brettterpstra.com/projects/searchlink/ for help'
      end
      print input
      Process.exit
    end

    @cfg['inline'] = true if input.scan(/\]\(/).length == 1 && input.split(/\n/).length == 1
    @errors = {}
    @report = []

    links = {}
    @footer = []
    counter_links = 0
    counter_errors = 0

    input.sub!(/\n?<!-- Report:.*?-->\n?/m, '')
    input.sub!(/\n?<!-- Errors:.*?-->\n?/m, '')

    input.scan(/\[(.*?)\]:\s+(.*?)\n/).each { |match| links[match[1].strip] = match[0] }

    prefix = if @cfg['prefix_random']
               if input =~ /\[(\d{4}-)\d+\]: \S+/
                 Regexp.last_match(1)
               else
                 format('%04d-', rand(9999))
               end
             else
               ''
             end

    highest_marker = 0
    input.scan(/^\s{,3}\[(?:#{prefix})?(\d+)\]: /).each do
      m = Regexp.last_match
      highest_marker = m[1].to_i if m[1].to_i > highest_marker
    end

    footnote_counter = 0
    input.scan(/^\s{,3}\[\^(?:#{prefix})?fn(\d+)\]: /).each do
      m = Regexp.last_match
      footnote_counter = m[1].to_i if m[1].to_i > footnote_counter
    end

    if input =~ /\[(.*?)\]\((.*?)\)/
      lines = input.split(/\n/)
      out = []

      total_links = input.scan(/\[(.*?)\]\((.*?)\)/).length
      in_code_block = false
      line_difference = 0
      lines.each_with_index do |line, num|
        @line_num = num - line_difference
        cursor_difference = 0
        # ignore links in code blocks
        if line =~ /^( {4,}|\t+)[^\*\+\-]/
          out.push(line)
          next
        end
        if line =~ /^[~`]{3,}/
          if in_code_block
            in_code_block = false
            out.push(line)
            next
          else
            in_code_block = true
          end
        end
        if in_code_block
          out.push(line)
          next
        end

        # line.gsub!(/\(\$ (.*?)\)/) do |match|
        #   this_match = Regexp.last_match
        #   match_column = this_match.begin(0)
        #   match_string = this_match.to_s
        #   match_before = this_match.pre_match
        #   match_after = this_match.post_match
        #   # todo: inline searches in larger context
        # end

        delete_line = false

        line.gsub!(/\[(.*?)\]\((.*?)\)/) do |match|

          this_match = Regexp.last_match
          @match_column = this_match.begin(0) - cursor_difference
          match_string = this_match.to_s
          @match_length = match_string.length
          match_before = this_match.pre_match

          invalid_search = false
          ref_title = false

          if match_before.scan(/(^|[^\\])`/).length % 2 == 1
            add_report("Match '#{match_string}' within an inline code block")
            invalid_search = true
          end

          counter_links += 1
          $stderr.print("\033[0K\rProcessed: #{counter_links} of #{total_links}, #{counter_errors} errors. ") unless SILENT

          link_text = this_match[1] || ''
          link_info = parse_arguments(this_match[2].strip).strip || ''

          if link_text.strip == '' && link_info =~ /".*?"/
            link_info.gsub!(/\"(.*?)\"/) {|q|
              link_text = $1 if link_text == ''
              $1
            }
          end

          if link_info.strip =~ /:$/ && line.strip == match
            ref_title = true
            link_info.sub!(/\s*:\s*$/,'')
          end

          unless link_text.length > 0 || link_info.sub(/^[!\^]\S+/,'').strip.length > 0
            add_error('No input', match)
            counter_errors += 1
            invalid_search = true
          end

          if link_info =~ /^!(\S+)/
            search_type = $1
            unless valid_search?(search_type) || search_type =~ /^(\S+\.)+\S+$/
              add_error('Invalid search', match)
              invalid_search = true
            end
          end


          if invalid_search
            match
          elsif link_info =~ /^\^(.+)/
            m = Regexp.last_match
            if m[1].nil? || m[1] == ''
              match
            else
              note = m[1].strip
              footnote_counter += 1
              ref = if !link_text.empty? && link_text.scan(/\s/).empty?
                      link_text
                    else
                      format("%<p>sfn%<c>04d", p: prefix, c: footnote_counter)
                    end
              add_footer "[^#{ref}]: #{note}"
              res = %Q{[^#{ref}]}
              cursor_difference = cursor_difference + (@match_length - res.length)
              @match_length = res.length
              add_report("#{match_string} => Footnote #{ref}")
              res
            end
          elsif (link_text == '' && link_info == '') || is_url?(link_info)
            add_error('Invalid search', match) unless is_url?(link_info)
            match
          else

            if link_text.length > 0 && link_info == ''
              link_info = link_text
            end

            search_type = ''
            search_terms = ''
            link_only = false
            @clipboard = false
            @titleize = @cfg['empty_uses_page_title']


            if link_info =~ /^(?:[!\^](\S+))\s*(.*)$/
              m = Regexp.last_match

              search_type = m[1].nil? ? 'g' : m[1]

              search_terms = m[2].gsub(/(^["']|["']$)/, '')
              search_terms.strip!

              # if the link text is just '%' replace with title regardless of config settings
              if link_text == '%' && search_terms && search_terms.length > 0
                @titleize = true
                link_text = ''
              end

              search_terms = link_text if search_terms == ''

              # if the input starts with a +, append it to the link text as the search terms
              search_terms = link_text + " " + search_terms.strip.sub(/^\+\s*/, '') if search_terms.strip =~ /^\+[^\+]/

              # if the end of input contain "^", copy to clipboard instead of STDOUT
              @clipboard = true if search_terms =~ /(!!)?\^(!!)?$/

              # if the end of input contains "!!", only print the url
              link_only = true if search_terms =~ /!!\^?$/

              search_terms.sub!(/(!!)?\^?(!!)?$/,"")

            elsif link_info =~ /^\!/
              search_word = link_info.match(/^\!(\S+)/)

              if search_word && valid_search?(search_word[1])
                search_type = search_word[1] unless search_word.nil?
                search_terms = link_text
              elsif search_word && search_word[1] =~ /^(\S+\.)+\S+$/
                search_type = 'g'
                search_terms = "site:#{search_word[1]} #{link_text}"
              else
                add_error('Invalid search', match)
                search_type = false
                search_terms = false
              end

            elsif link_text && link_text.length > 0 && (link_info.nil? || link_info.length == 0)

              search_type = 'g'
              search_terms = link_text
            else
              add_error('Invalid search', match)
              search_type = false
              search_terms = false
            end

            @cfg['custom_site_searches'].each do |k, v|
              if search_type == k
                link_text = search_terms if !@titleize && link_text == ''
                v = parse_arguments(v, { no_restore: true })
                if v =~ /^(\/|http)/i
                  search_type = 'r'
                  tokens = v.scan(/\$term\d+[ds]?/).sort.uniq

                  if tokens.length > 0
                    highest_token = 0
                    tokens.each {|token|
                      if token =~ /(\d+)[ds]?$/
                        highest_token = $1.to_i if $1.to_i > highest_token
                      end
                    }
                    terms_p = search_terms.split(/ +/)
                    if terms_p.length > highest_token
                      remainder = terms_p[highest_token-1..-1].join(" ")
                      terms_p = terms_p[0..highest_token - 2]
                      terms_p.push(remainder)
                    end
                    tokens.each do |t|
                      next unless t =~ /(\d+)[ds]?$/

                      int = Regexp.last_match(1).to_i - 1
                      replacement = terms_p[int]
                      case t
                      when /d$/
                        replacement.downcase!
                        re_down = ''
                      when /s$/
                        replacement.slugify!
                        re_down = ''
                      else
                        re_down = '(?!d|s)'
                      end
                      v.gsub!(/#{Regexp.escape(t) + re_down}/, ERB::Util.url_encode(replacement))
                    end
                    search_terms = v
                  else
                    search_terms = v.gsub(/\$term[ds]?/i) do |mtch|
                      search_terms.downcase! if mtch =~ /d$/i
                      search_terms.slugify! if mtch =~ /s$/i
                      ERB::Util.url_encode(search_terms)
                    end
                  end
                else
                  search_type = 'g'
                  search_terms = "site:#{v} #{search_terms}"
                end

                break
              end
            end if search_type && !search_terms.empty?

            if search_type && search_terms
              # $stderr.puts "Searching #{search_type} for #{search_terms}"
              url, title, link_text = do_search(search_type, search_terms, link_text)

              if url
                title = titleize(url) if @titleize && title == ''

                link_text = title if link_text == '' && title
                force_title = search_type =~ /def/ ? true : false

                if link_only || search_type =~ /sp(ell)?/ || url == 'embed'
                  url = title if url == 'embed'
                  cursor_difference += @match_length - url.length
                  @match_length = url.length
                  add_report("#{match_string} => #{url}")
                  url
                elsif ref_title
                  unless links.key? url
                    links[url] = link_text
                    add_footer make_link('ref_title', link_text, url, title: title, force_title: force_title)
                  end
                  delete_line = true
                elsif @cfg['inline']
                  res = make_link('inline', link_text, url, title: title, force_title: force_title)
                  cursor_difference += @match_length - res.length
                  @match_length = res.length
                  add_report("#{match_string} => #{url}")
                  res
                else
                  unless links.key? url
                    highest_marker += 1
                    links[url] = format('%<pre>s%<m>04d', pre: prefix, m: highest_marker)
                    add_footer make_link('ref_title', links[url], url, title: title, force_title: force_title)
                  end

                  type = @cfg['inline'] ? 'inline' : 'ref_link'
                  res = make_link(type, link_text, links[url], title: false, force_title: force_title)
                  cursor_difference += @match_length - res.length
                  @match_length = res.length
                  add_report("#{match_string} => #{url}")
                  res
                end
              else
                add_error('No results', "#{search_terms} (#{match_string})")
                counter_errors += 1
                match
              end
            else
              add_error('Invalid search', match)
              counter_errors += 1
              match
            end
          end
        end
        line_difference += 1 if delete_line
        out.push(line) unless delete_line
        delete_line = false
      end
      $stderr.puts("\n") unless SILENT

      input = out.delete_if {|l|
        l.strip =~ /^<!--DELETE-->$/
      }.join("\n")

      if @cfg['inline']
        add_output input + "\n"
        add_output "\n" + print_footer unless @footer.empty?
      else
        if @footer.empty?
          add_output input
        else
          last_line = input.strip.split(/\n/)[-1]
          if last_line =~ /^\[.*?\]: http/
            add_output input.rstrip + "\n"
          elsif last_line =~ /^\[\^.*?\]: /
            add_output input.rstrip
          else
            add_output input + "\n\n"
          end
          add_output print_footer + "\n\n"
        end
      end
      @line_num = nil
      add_report("Processed: #{total_links} links, #{counter_errors} errors.")
      print_report
      print_errors
    else
      link_only = false
      @clipboard = false

      input = parse_arguments(input.strip!).strip

      # if the end of input contain "^", copy to clipboard instead of STDOUT
      @clipboard = true if input =~ /\^[!~:]*$/

      # if the end of input contains "!!", only print the url
      link_only = true if input =~ /!![\^~:]*$/

      reference_link = input =~ /:([!\^\s~]*)$/

      # if end of input contains ~, pull url from clipboard
      if input =~ /~[:\^!\s]*$/
        input.sub!(/[:!\^\s~]*$/,'')
        clipboard = %x{__CF_USER_TEXT_ENCODING=$UID:0x8000100:0x8000100 pbpaste}.strip
        if is_url?(clipboard)
          type = reference_link ? 'ref_title' : 'inline'
          print make_link(type, input.strip, clipboard)
        else
          print @originput
        end
        Process.exit
      end

      input.sub!(/[:!\^\s~]*$/,'')

      ## Maybe if input is just a URL, convert it to a link
      ## using hostname as text without doing search
      # if only_url?(input.strip)
      #   url, title = url_to_link(input.strip)
      #   type = reference_link ? 'ref_title' : 'inline'
      #   print make_link(type, title, url, false, false)
      #   Process.exit
      # end

      # check for additional search terms in parenthesis
      additional_terms = ''
      if input =~ /\((.*?)\)/
        additional_terms = " " + $1.strip
        input.sub!(/\(.*?\)/,'')
      end

      link_text = false

      if input =~ /"(.*?)"/
        link_text = $1
        input.gsub!(/"(.*?)"/, '\1')
      end

      # remove quotes from terms, just in case
      # input.sub!(/^(!\S+)?\s*(["'])(.*?)\2([\!\^]+)?$/, "\\1 \\3\\4")

      if input =~ /^!(\S+)\s+(.*)$/
        type = $1
        link_info = $2.strip
        link_text = link_info unless link_text
        terms = link_info  + additional_terms
        terms.strip!

        if valid_search?(type) || type =~ /^(\S+\.)+\S+$/
          @cfg['custom_site_searches'].each {|k,v|
            if type == k
              link_text = terms if link_text == ''
              v = parse_arguments(v, {:no_restore => true})
              if v =~ /^(\/|http)/i
                type = 'r'
                tokens = v.scan(/\$term\d+[ds]?/).sort.uniq

                if tokens.length > 0
                  highest_token = 0
                  tokens.each {|token|
                    if token =~ /(\d+)d?$/
                      highest_token = $1.to_i if $1.to_i > highest_token
                    end
                  }
                  terms_p = terms.split(/ +/)
                  if terms_p.length > highest_token
                    remainder = terms_p[highest_token-1..-1].join(" ")
                    terms_p = terms_p[0..highest_token - 2]
                    terms_p.push(remainder)
                  end
                  tokens.each do |t|
                    next unless t =~ /(\d+)d?$/

                    int = Regexp.last_match(1).to_i - 1
                    replacement = terms_p[int]
                    case t
                    when /d$/
                      replacement.downcase!
                      re_down = ''
                    when /s$/
                      replacement.slugify!
                      re_down = ''
                    else
                      re_down = '(?!d|s)'
                    end
                    v.gsub!(/#{Regexp.escape(t) + re_down}/, ERB::Util.url_encode(replacement))
                  end
                  terms = v
                else
                  terms = v.gsub(/\$term[ds]?/i) do |mtch|
                    terms.downcase! if mtch =~ /d$/i
                    terms.slugify! if mtch =~ /s$/i
                    ERB::Util.url_encode(terms)
                  end
                end

              else
                type = 'g'
                terms = "site:#{v} #{terms}"
              end

              break
            end
          } if type && terms && terms.length > 0

          if type =~ /^(\S+\.)+\S+$/
            terms = "site:#{type} #{terms}"
            type = 'g'
          end

          url, title, link_text = do_search(type, terms, link_text)
        else
          add_error('Invalid search', input)
          counter_errors += 1
        end
      elsif input =~ /^([tfil])?@(\S+)\s*$/
        type = $1 || 't'
        link_text = input.sub(/^[tfil]/,'')
        url, title = social_handle(type, link_text)
        link_text = title
      else
        link_text = input unless link_text
        url, title = ddg(input)
      end

      if url
        if type =~ /sp(ell)?/
          add_output(url)
        elsif link_only
          add_output(url)
        elsif url == 'embed'
          add_output(title)
        else
          type = reference_link ? 'ref_title' : 'inline'
          add_output make_link(type, link_text, url, title: title, force_title: false)
          print_errors
        end
      else
        add_error('No results', title)
        add_output @originput.chomp
        print_errors
      end

      if @clipboard
        if @output == @originput
          $stderr.puts "No results found"
        else
          %x{echo #{Shellwords.escape(@output)}|tr -d "\n"|pbcopy}
          $stderr.puts "Results in clipboard"
        end
      end
    end
  end

  private

  def parse_arguments(string, opt={})
    input = string.dup
    skip_flags = opt[:only_meta] || false
    no_restore = opt[:no_restore] || false
    restore_prev_config unless no_restore

    input.gsub!(/(\+\+|--)([dirtv]+)\b/) do
      m = Regexp.last_match
      bool = m[1] == '++' ? '' : 'no-'
      output = ' '
      m[2].split('').each do |arg|
        output += case arg
                  when 'd'
                    "--#{bool}debug "
                  when 'i'
                    "--#{bool}inline "
                  when 'r'
                    "--#{bool}prefix_random "
                  when 't'
                    "--#{bool}include_titles "
                  when 'v'
                    "--#{bool}validate_links "
                  else
                    ''
                  end
      end
      output
    end unless skip_flags

    options = %w[debug country_code inline prefix_random include_titles validate_links]
    options.each do |o|
      if input =~ /^#{o}:\s+(.*?)$/
        val = Regexp.last_match(1).strip
        val = true if val =~ /true/i
        val = false if val =~ /false/i
        @cfg[o] = val
        $stderr.print "\r\033[0KGlobal config: #{o} = #{@cfg[o]}\n" unless SILENT
      end

      next if skip_flags

      while input =~ /^#{o}:\s+(.*?)$/ || input =~ /--(no-)?#{o}/
        next unless input =~ /--(no-)?#{o}/ && !skip_flags

        unless @prev_config.key? o
          @prev_config[o] = @cfg[o]
          bool = Regexp.last_match(1).nil? || Regexp.last_match(1) == '' ? true : false
          @cfg[o] = bool
          $stderr.print "\r\033[0KLine config: #{o} = #{@cfg[o]}\n" unless SILENT
        end
        input.sub!(/\s?--(no-)?#{o}/, '')
      end
    end
    @clipboard ? string : input
  end

  def restore_prev_config
    @prev_config&.each do |k, v|
      @cfg[k] = v
      $stderr.print "\r\033[0KReset config: #{k} = #{@cfg[k]}\n" unless SILENT
    end
    @prev_config = {}
  end

  def make_link(type, text, url, title: false, force_title: false)
    text = title ? title : titleize(url) if @titleize && text == ''

    title = title && (@cfg['include_titles'] || force_title) ? %Q{ "#{title.clean}"} : ""

    case type
    when 'ref_title'
      %Q{\n[#{text.strip}]: #{url}#{title}}
    when 'ref_link'
      %Q{[#{text.strip}][#{url}]}
    when 'inline'
      %Q{[#{text.strip}](#{url}#{title})}
    end
  end

  def add_output(str)
    if @printout && !@clipboard
      print str
    end
    @output += str
  end

  def add_footer(str)
    @footer ||= []
    @footer.push(str.strip)
  end

  def print_footer
    unless @footer.empty?

      footnotes = []
      @footer.delete_if {|note|
        note.strip!
        if note =~ /^\[\^.+?\]/
          footnotes.push(note)
          true
        elsif note =~ /^\s*$/
          true
        else
          false
        end
      }

      output = @footer.sort.join("\n").strip
      output += "\n\n" if output.length > 0 && !footnotes.empty?
      output += footnotes.join("\n\n") unless footnotes.empty?
      return output.gsub(/\n{3,}/,"\n\n")
    end
    return ""
  end

  def add_report(str)
    if @cfg['report']
      unless @line_num.nil?
        position = @line_num.to_s + ':'
        position += @match_column.nil? ? "0:" : "#{@match_column}:"
        position += @match_length.nil? ? "0" : @match_length.to_s
      end
      @report.push("(#{position}): #{str}")
      $stderr.puts "(#{position}): #{str}" unless SILENT
    end
  end

  def add_error(type, str)
    if @cfg['debug']
      unless @line_num.nil?
        position = @line_num.to_s + ':'
        position += @match_column.nil? ? "0:" : "#{@match_column}:"
        position += @match_length.nil? ? "0" : @match_length.to_s
      end
      @errors[type] ||= []
      @errors[type].push("(#{position}): #{str}")
    end
  end

  def print_report
    return if (@cfg['inline'] && @originput.split(/\n/).length == 1) || @clipboard
    unless @report.empty?
      out = "\n<!-- Report:\n#{@report.join("\n")}\n-->\n"
      add_output out
    end
  end

  def print_errors(type = 'Errors')
    return if @errors.empty?
    out = ''
    if @originput.split(/\n/).length > 1
      inline = false
    else
      inline = @cfg['inline'] || @originput.split(/\n/).length == 1
    end

    @errors.each {|k,v|
      unless v.empty?
        v.each_with_index {|err, i|
          out += "(#{k}) #{err}"
          out += inline ? i == v.length - 1 ? " | " : ", " : "\n"
        }
      end
    }
    unless out == ''
      sep = inline ? " " : "\n"
      out.sub!(/\| /, '')
      out = "#{sep}<!-- #{type}:#{sep}#{out}-->#{sep}"
    end
    if @clipboard
      $stderr.puts out
    else
      add_output out
    end
  end

  def print_or_copy(text)
    # Process.exit unless text
    if @clipboard
      %x{echo #{Shellwords.escape(text)}|tr -d "\n"|pbcopy}
      print @originput
    else
      print text
    end
  end

  def notify(str, sub)
    return unless @cfg['notifications']
    %x{osascript -e 'display notification "SearchLink" with title "#{str}" subtitle "#{sub}"'}
  end

  def valid_link?(uri_str, limit = 5)
    begin
      notify("Validating", uri_str)
      return false if limit == 0
      url = URI(uri_str)
      return true unless url.scheme
      if url.path == ""
        url.path = "/"
      end
      # response = Net::HTTP.get_response(URI(uri_str))
      response = false

      Net::HTTP.start(url.host, url.port, :use_ssl => url.scheme == 'https') {|http| response = http.request_head(url.path) }

      case response
      when Net::HTTPMethodNotAllowed, Net::HTTPServiceUnavailable then
        unless /amazon\.com/ =~ url.host
          add_error('link validation', "Validation blocked: #{uri_str} (#{e})")
        end
        notify("Error validating", uri_str)
        true
      when Net::HTTPSuccess then
        true
      when Net::HTTPRedirection then
        location = response['location']
        valid_link?(location, limit - 1)
      else
        notify("Error validating", uri_str)
        false
      end
    rescue => e
      notify("Error validating", uri_str)
      add_error('link validation', "Possibly invalid => #{uri_str} (#{e})")
      return true
    end
  end

  def is_url?(input)
    input =~ /^(#.*|https?:\/\/\S+|\/\S+|\S+\/|[^!]\S+\.\S+)(\s+".*?")?$/
  end

  def only_url?(input)
    input =~ /(?i)^((http|https):\/\/)?([\w\-_]+(\.[\w\-_]+)+)([\w\-\.,@?^=%&amp;:\/~\+#]*[\w\-\@^=%&amp;\/~\+#])?$/
  end

  def url_to_link(input)
    if (only_url?(input))
      input.sub!( /(?mi)^(?!https?:\/\/)(.*?)$/, "https://\\1" )
      url = URI.parse(input.downcase)
      if (url.hostname)

        return [url.to_s, input.sub(/^https?:\/\//,'')]
      end
    end
    false
  end

  def valid_search?(term)
    valid = false
    valid = true if term =~ /(^h(([sc])([hb])?)*|^a$|^imov|^g$|^b$|^wiki$|^def$|^masd?$|^itud?$|^s$|^(i|am)(art|alb|song|pod)e?$|^lart|^@(t|f|i|l)|^r$|^sp(ell)?|pb$)/
    valid = true if @cfg['custom_site_searches'].keys.include? term
    notify("Invalid search", term) unless valid
    valid
  end

  def search_chrome_history(term)
    # Google history
    if File.exists?(File.expand_path('~/Library/Application Support/Google/Chrome/Default/History'))
      notify("Searching Chrome History", term)
      tmpfile = File.expand_path('~/Library/Application Support/Google/Chrome/Default/History.tmp')
      FileUtils.cp(File.expand_path('~/Library/Application Support/Google/Chrome/Default/History'), tmpfile)

      terms = []
      terms.push("(url NOT LIKE '%search/?%' AND url NOT LIKE '%?q=%' AND url NOT LIKE '%?s=%')")
      terms.concat(term.split(/\s+/).map {|t| "(url LIKE '%#{t.strip.downcase}%' OR title LIKE '%#{t.strip.downcase}%')" })
      query = terms.join(" AND ")
      most_recent = %x{sqlite3 -separator ' ;;; ' '#{tmpfile}' "select title,url,datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') from urls where #{query} AND NOT (url LIKE '%?s=%' OR url LIKE '%/search%') order by last_visit_time limit 1 COLLATE NOCASE;"}.strip
      FileUtils.rm_f(tmpfile)
      return false if most_recent.strip.length == 0
      title, url, date = most_recent.split(/\s*;;; /)
      date = Time.parse(date)
      [url, title, date]
    else
      false
    end
  end

  def search_chrome_bookmarks(term)
    out = false
    if File.exists?(File.expand_path('~/Library/Application Support/Google/Chrome/Default/Bookmarks'))
      notify("Searching Chrome Bookmarks", term)
      chrome_bookmarks = JSON.parse(IO.read(File.expand_path('~/Library/Application Support/Google/Chrome/Default/Bookmarks')))
      if chrome_bookmarks
        terms = term.split(/\s+/)
        roots = chrome_bookmarks['roots']
        urls = extract_chrome_bookmarks(roots)
        urls.sort_by! {|bookmark| bookmark["date_added"]}
        urls.select {|u|
          found = false
          terms.each {|t|
            if u["url"] =~ /#{t}/i || u["title"] =~ /#{t}/
              found = true
            end
          }
          found
        }
        unless urls.empty?
          lastest_bookmark = urls[-1]
          out = [lastest_bookmark['url'], lastest_bookmark['title'], lastest_bookmark['date']]
        end
      end
    end
    out
  end

  def search_history(term,types = [])
    if types.empty?
      if @cfg['history_types']
        types = @cfg['history_types']
      else
        return false
      end
    end


    results = []

    if types.length > 0
      types.each {|type|

        url, title, date = case type
        when 'chrome_history'
          search_chrome_history(term)
        when 'chrome_bookmarks'
          search_chrome_bookmarks(term)
        when 'safari_bookmarks'
          search_safari_urls(term, 'bookmark')
        when 'safari_history'
          search_safari_urls(term, 'history')
        when 'safari_all'
          search_safari_urls(term)
        else
          false
        end
        if url
          results << {'url' => url, 'title' => title, 'date' => date}
        end
      }

      unless results.empty?
        out = results.sort_by! {|r| r['date'] }[-1]
        [out['url'], out['title']]
      else
        false
      end
    else
      false
    end
  end

  # FIXME: These spotlight searches no longer work on 10.13
  # Search Safari Bookmarks and/or history using spotlight
  #
  # @param (String) term
  # @param (String) type ['history'|'bookmark'|(Bool) false]
  #
  # @return (Array) [url, title, access_date] on success
  # @return (Bool) false on error
  def search_safari_urls(term,type = false)
    notify("Searching Safari History", term)
    onlyin = "~/Library/Caches/Metadata/Safari"
    onlyin += type ? "/"+type.capitalize : "/"
    type = type ? ".#{type}" : "*"
    # created:>10/13/13 kind:safari filename:.webbookmark
    # Safari history/bookmarks
    terms = term.split(/\s+/).delete_if {|t| t.strip =~ /^\s*$/ }.map{|t|
      %Q{kMDItemTextContent = "*#{t}*"cdw}
    }.join(" && ")

    date = type == ".history" ? "&& kMDItemContentCreationDate > $time.today(-182) " : ""

    avoid_results = ["404", "not found", "chrome-extension"].map {|q|
      %Q{ kMDItemDisplayName != "*#{q}*"cdw }
    }.join(" && ")
    query = %Q{((kMDItemContentType = "com.apple.safari#{type}") #{date}&& (#{avoid_results}) && (#{terms}))}

    search = %x{mdfind -onlyin #{onlyin.gsub(/ /,'\ ')} '#{query}'}
    if search.length > 0
      res = []
      search.split(/\n/).each {|file|
        url = %x{mdls -raw -name kMDItemURL "#{file}"}
        date = %x{mdls -raw -name kMDItemDateAdded "#{file}"}
        date = Time.parse(date)
        title = %x{mdls -raw -name kMDItemDisplayName "#{file}"}
        res << {'url' => url, 'date' => date, 'title' => title}
      }
      res.delete_if {|k,el| el =~ /\(null\)/ }

      latest = res.sort_by! {|r| r["date"] }[-1]
      [latest['url'], latest['title'], latest['date']]
    else
      false
    end
  end

  def extract_chrome_bookmarks(json,urls = [])

    if json.class == Array
      json.each {|item|
        urls = extract_chrome_bookmarks(item, urls)
      }
    elsif json.class == Hash
      if json.has_key? "children"
        urls = extract_chrome_bookmarks(json["children"],urls)
      elsif json["type"] == "url"
        date = Time.at(json["date_added"].to_i / 1000000 + (Time.new(1601,01,01).strftime('%s').to_i))
        urls << {'url' => json["url"], 'title' => json["name"], 'date' => date}
      else
        json.each {|k,v|
          urls = extract_chrome_bookmarks(v,urls)
        }

      end
    else
      return urls
    end
    urls
  end



  def wiki(terms)
    ## Hack to scrape wikipedia result
    body = %x{/usr/bin/curl -sSL 'https://en.wikipedia.org/wiki/Special:Search?search=#{ERB::Util.url_encode(terms)}&go=Go'}
    if body
      if RUBY_VERSION.to_f > 1.9
        body = body.force_encoding('utf-8')
      end

      begin
        title = body.match(/"wgTitle":"(.*?)"/)[1]
        url = body.match(/<link rel="canonical" href="(.*?)"/)[1]
      rescue
        return false
      end
      return [url, title]
    end
    ## Removed because Ruby 2.0 does not like https connection to wikipedia without using gems?
    # uri = URI.parse("https://en.wikipedia.org/w/api.php?action=query&format=json&prop=info&inprop=url&titles=#{CGI.escape(terms)}")
    # req = Net::HTTP::Get.new(uri.path)
    # req['Referer'] = "http://brettterpstra.com"
    # req['User-Agent'] = "SearchLink (http://brettterpstra.com)"

    # res = Net::HTTP.start(uri.host, uri.port,
    #   :use_ssl => true,
    #   :verify_mode => OpenSSL::SSL::VERIFY_NONE) do |https|
    #     https.request(req)
    #   end



    # if RUBY_VERSION.to_f > 1.9
    #   body = res.body.force_encoding('utf-8')
    # else
    #   body = res.body
    # end

    # result = JSON.parse(body)

    # if result
    #   result['query']['pages'].each do |page,info|
    #     unless info.key? "missing"
    #       return [info['fullurl'],info['title']]
    #     end
    #   end
    # end
    # return false
  end

  def zero_click(terms)
    url = URI.parse("http://api.duckduckgo.com/?q=#{ERB::Util.url_encode(terms)}&format=json&no_redirect=1&no_html=1&skip_disambig=1")
    res = Net::HTTP.get_response(url).body
    res = res.force_encoding('utf-8') if RUBY_VERSION.to_f > 1.9

    result = JSON.parse(res)
    if result
      definition = result['Definition'] || false
      definition_link = result['DefinitionURL'] || false
      wiki_link = result['AbstractURL'] || false
      title = result['Heading'] || false
      return [title, definition, definition_link, wiki_link]
    else
      return false
    end
  end

  # Search apple music
  # terms => search terms (unescaped)
  # media => music, podcast
  # entity => optional: artist, song, album, podcast
  # returns {:type=>,:id=>,:url=>,:title}
  def applemusic(terms, media='music', entity='')
    aff = @cfg['itunes_affiliate']
    output = {}

    url = URI.parse("http://itunes.apple.com/search?term=#{ERB::Util.url_encode(terms)}&country=#{@cfg['country_code']}&media=#{media}&entity=#{entity}")
    res = Net::HTTP.get_response(url).body
    res = res.force_encoding('utf-8') if RUBY_VERSION.to_f > 1.9
    res.gsub!(/(?mi)[\x00-\x08\x0B-\x0C\x0E-\x1F]/,'')
    json = JSON.parse(res)
    if json['resultCount'] && json['resultCount'] > 0
      result = json['results'][0]

      case result['wrapperType']
      when 'track'
        if result['kind'] == 'podcast'
          output[:type] = 'podcast'
          output[:id] = result['collectionId']
          output[:url] = result['collectionViewUrl'].to_am + aff
          output[:title] = result['collectionName']
        else
          output[:type] = 'song'
          output[:album] = result['collectionId']
          output[:id] = result['trackId']
          output[:url] = result['trackViewUrl'].to_am + aff
          output[:title] = result['trackName'] + " by " + result['artistName']
        end
      when 'collection'
        output[:type] = 'album'
        output[:id] = result['collectionId']
        output[:url] = result['collectionViewUrl'].to_am + aff
        output[:title] = result['collectionName'] + " by " + result['artistName']
      when 'artist'
        output[:type] = 'artist'
        output[:id] = result['artistId']
        output[:url] = result['artistLinkUrl'].to_am + aff
        output[:title] = result['artistName']
      end
      return false if output.empty?
      output
    else
      return false
    end
  end

  def itunes(entity, terms, dev, aff='')
    aff = @cfg['itunes_affiliate']

    url = URI.parse("http://itunes.apple.com/search?term=#{ERB::Util.url_encode(terms)}&country=#{@cfg['country_code']}&entity=#{entity}")
    res = Net::HTTP.get_response(url).body
    res = res.force_encoding('utf-8').encode # if RUBY_VERSION.to_f > 1.9

    begin
      json = JSON.parse(res)
    rescue => e
      add_error('Invalid response', "Search for #{terms}: (#{e})")
      return false
    end
    return false unless json
    if json['resultCount'] && json['resultCount'] > 0
      result = json['results'][0]
      case entity
      when /movie/
        # dev parameter probably not necessary in this case
        output_url = result['trackViewUrl']
        output_title = result['trackName']
      when /(mac|iPad)Software/
        output_url = dev && result['sellerUrl'] ? result['sellerUrl'] : result['trackViewUrl']
        output_title = result['trackName']
      when /(musicArtist|song|album)/
        case result['wrapperType']
        when 'track'
          output_url = result['trackViewUrl']
          output_title = result['trackName'] + " by " + result['artistName']
        when 'collection'
          output_url = result['collectionViewUrl']
          output_title = result['collectionName'] + " by " + result['artistName']
        when 'artist'
          output_url = result['artistLinkUrl']
          output_title = result['artistName']
        end
      when /podcast/
        output_url = result['collectionViewUrl']
        output_title = result['collectionName']
      end
      return false unless output_url and output_title
      if dev
        return [output_url, output_title]
      else
        return [output_url + aff, output_title]
      end
    else
      return false
    end
  end

  def lastfm(entity, terms)
    url = URI.parse("http://ws.audioscrobbler.com/2.0/?method=#{entity}.search&#{entity}=#{ERB::Util.url_encode(terms)}&api_key=2f3407ec29601f97ca8a18ff580477de&format=json")
    res = Net::HTTP.get_response(url).body
    res = res.force_encoding('utf-8') if RUBY_VERSION.to_f > 1.9
    json = JSON.parse(res)
    if json['results']
      begin
        case entity
        when 'track'
          result = json['results']['trackmatches']['track'][0]
          url = result['url']
          title = result['name'] + " by " + result['artist']
        when 'artist'
          result = json['results']['artistmatches']['artist'][0]
          url = result['url']
          title = result['name']
        end
        return [url, title]
      rescue
        return false
      end
    else
      return false
    end
  end

  def define(terms)
    begin
      def_url = "https://www.wordnik.com/words/#{ERB::Util.url_encode(terms)}"
      body = %x{/usr/bin/curl -sSL '#{def_url}'}
      if body =~ /id="define"/
        first_definition = body.match(/(?mi)(?:id="define"[\s\S]*?<li>)([\s\S]*?)<\/li>/)[1]
        parts = first_definition.match(/<abbr title="partOfSpeech">(.*?)<\/abbr> (.*?)$/)
        return [def_url, "(#{parts[1]}) #{parts[2]}"]
      end
      return false
    rescue
      return false
    end
  end

  def pinboard_bookmarks
    bookmarks = %x{/usr/bin/curl -sSL "https://api.pinboard.in/v1/posts/all?auth_token=#{@cfg['pinboard_api_key']}&format=json"}
    bookmarks = bookmarks.force_encoding('utf-8')
    bookmarks.gsub!(/[^[:ascii:]]/) do |non_ascii|
      non_ascii.force_encoding('utf-8')
               .encode('utf-16be')
               .unpack('H*').first
               .gsub(/(....)/,'\u\1')
    end

    bookmarks.gsub!(/[\u{1F600}-\u{1F6FF}]/,'')

    bookmarks = JSON.parse(bookmarks)
    updated = Time.now
    result = {'update_time' => updated, 'bookmarks' => bookmarks}
    result
  end

  def save_pinboard_cache(cache)
    cachefile = PINBOARD_CACHE

    # file = File.new(cachefile,'w')
    # file = Zlib::GzipWriter.new(File.new(cachefile,'w'))
    begin
      marshal_dump = Marshal.dump(cache)
      File.write(cachefile, marshal_dump)
    rescue IOError => e
      add_error("Pinboard cache error","Failed to write stash to disk")
      return false
    end
    return true
  end

  def get_pinboard_cache
    refresh_cache = false
    cachefile = PINBOARD_CACHE

    if File.exists?(cachefile)
      begin
        file = IO.read(cachefile) # Zlib::GzipReader.open(cachefile)
        cache = Marshal.load file
        # file.close
      rescue IOError => e # Zlib::GzipFile::Error
        add_error("Error loading pinboard cache","Error reading #{cachefile}")
        return false
      end
      updated = JSON.parse(%x{/usr/bin/curl -sSL 'https://api.pinboard.in/v1/posts/update?auth_token=#{@cfg['pinboard_api_key']}&format=json'})
      last_bookmark = Time.parse(updated['update_time'])
      if cache && cache.key?('update_time')
        last_update = Time.parse(cache['update_time'])
        if last_update < last_bookmark
          refresh_cache = true
        end
      else
        refresh_cache = true
      end
    else
      refresh_cache = true
    end

    if refresh_cache
      cache = pinboard_bookmarks
      save_pinboard_cache(cache)
    end

    return cache
  end

  def pinboard(terms)
    unless @cfg['pinboard_api_key']
      add_error('Missing Pinboard API token', "Find your api key at https://pinboard.in/settings/password and add it to your configuration (pinboard_api_key: YOURKEY)")
      return false
    end

    result = false

    regex = terms.split(/ /).map { |arg| Regexp.escape arg }.join(".*?")
    regex = /#{regex}/i

    # cache = get_pinboard_cache
    cache = pinboard_bookmarks
    bookmarks = cache['bookmarks']

    bookmarks.each {|bm|
      text = [bm['description'],bm['tags']].join(" ")
      if text =~ regex
        result = [bm['href'],bm['description']]
        break
      end
    }
    result
  end

  # Search bookmark paths and addresses. Return array of bookmark hashes.
  def search_hook(search)
    path_matches = `osascript <<'APPLESCRIPT'
      set searchString to "#{search.strip}"
      tell application "Hook"
        set _marks to every bookmark whose name contains searchString or path contains searchString or address contains searchString
        set _out to {}
        repeat with _hook in _marks
          set _out to _out & (name of _hook & "||" & address of _hook & "||" & path of _hook)
        end repeat
        set {astid, AppleScript's text item delimiters} to {AppleScript's text item delimiters, "^^"}
        set _output to _out as string
        set AppleScript's text item delimiters to astid
        return _output
      end tell
    APPLESCRIPT`.strip.split_hooks

    top_match = path_matches.uniq.first
    return false unless top_match

    [top_match[:url], top_match[:name]]
  end

  def google(terms, define = false)
    begin
      uri = URI.parse("http://ajax.googleapis.com/ajax/services/search/web?v=1.0&filter=1&rsz=small&q=#{ERB::Util.url_encode(terms)}")
      req = Net::HTTP::Get.new(uri.request_uri)
      req['Referer'] = "http://brettterpstra.com"
      res = Net::HTTP.start(uri.host, uri.port) {|http|
        http.request(req)
      }
      if RUBY_VERSION.to_f > 1.9
        body = res.body.force_encoding('utf-8')
      else
        body = res.body
      end

      json = JSON.parse(body)
      if json['responseData']
        result = json['responseData']['results'][0]
        return false if result.nil?
        output_url = result['unescapedUrl']
        if define && output_url =~ /dictionary/
          output_title = result['content'].gsub(/<\/?.*?>/,'')
        else
          output_title = result['titleNoFormatting']
        end
        return [output_url, output_title]
      else
        return ddg(terms, false)
      end
    rescue
      return ddg(terms, false)
    end
  end

  def ddg(terms,type = false)
    prefix = type ? "#{type.sub(/^!?/, '!')} " : '%5C'

    begin
      body = `/usr/bin/curl -LisS 'https://lite.duckduckgo.com/lite/?q=#{prefix}#{ERB::Util.url_encode(terms)}'`
      locs = body.scan(/^location: (.*?)$/)
      return false unless locs

      url = locs[-1]
      result = url[0].strip || false
      return false unless result

      output_url = CGI.unescape(result)
      output_title = if @cfg['include_titles'] || @titleize
                       titleize(output_url) || ''
                     else
                       ''
                     end
      [output_url, output_title]
    end
  end

  def titleize(url)
    whitespace = url.match(/(\s*$)/)[0] || ''
    title = nil
    begin
      # source = %x{/usr/bin/curl -sSL '#{url.strip}'}

      uri = URI.parse(url)
      res = Net::HTTP.get_response(uri)

      if res.code.to_i == 200
        source = res.body
        unless source
          title = nil
        else
          title = source.match(/<title>(.*)<\/title>/im)
        end

        title = title.nil? ? nil : title[1].strip
      end

      if title.nil? || title =~ /^\s*$/
        $stderr.puts "Warning: missing title for #{url.strip}"
        title = url.gsub(/(^https?:\/\/|\/.*$)/,'').gsub(/-/,' ').strip
      else
        title = title.gsub(/\n/, ' ').gsub(/\s+/,' ').strip # .sub(/[^a-z]*$/i,'')
      end

      # Skipping SEO removal until it's more reliable
      # title.remove_seo(url.strip)
      title
    rescue Exception => e
      $stderr.puts "Error retrieving title for #{url.strip}"
      raise e
    end
  end

  def spell(phrase)
    unless File.exists?("/usr/local/bin/aspell")
      add_error('Missing aspell', "Install aspell in /usr/local/bin/aspell to allow spelling corrections")
      return false
    end
    words = phrase.split(/\b/)
    output = ""
    words.each do |w|
      if w =~ /[A-Za-z]+/
        spell_res =  `echo "#{w}" | /usr/local/bin/aspell --sug-mode=bad-spellers -C pipe | head -n 2 | tail -n 1`
        if spell_res.strip == "\*"
          output += w
        else
          spell_res.sub!(/.*?: /,'')
          results = spell_res.split(/, /).delete_if { |w| phrase =~ /^[a-z]/ && w =~ /[A-Z]/ }
          output += results[0]
        end
      else
        output += w
      end
    end
    output
  end

  def amazon_affiliatize(url, amazon_partner)
    return url if amazon_partner.nil? || amazon_partner.length == 0

    if url =~ /https?:\/\/(?:.*?)amazon.com\/(?:(.*?)\/)?([dg])p\/([^\?]+)/
      title = $1
      type = $2
      id = $3
      az_url = "http://www.amazon.com/#{type}p/product/#{id}/ref=as_li_ss_tl?ie=UTF8&linkCode=ll1&tag=#{amazon_partner}"
      return [az_url, title]
    else
      return [url,'']
    end
  end

  def template_social(user, url, service)
    template = @cfg['social_template']
    template.sub!(/%user%/, user)
    template.sub!(/%service%/, service)
    template.sub!(/%url%/, url.sub(/^https?:\/\/(www\.)?/,'').sub(/\/$/,''))
    template
  end

  def social_handle(type, term)
    handle = term.sub(/^@/,'').strip
    case type
    when /^t/
      url = "https://twitter.com/#{handle}"
      title = template_social(handle, url, 'Twitter')
    when /^f/
      url = "https://www.facebook.com/#{handle}"
      title = template_social(handle, url, 'Facebook')
    when /^l/
      url = "https://www.linkedin.com/in/#{handle}/"
      title = template_social(handle, url, 'LinkedIn')
    when /^i/
      url = "https://www.instagram.com/#{handle}/"
      title = template_social(handle, url, 'Instagram')
    else
      [false, term, link_text]
    end
    [url, title]
  end

  def do_search(search_type, search_terms, link_text='')
    notify("Searching", search_terms)
    return [false, search_terms, link_text] unless search_terms.length > 0

    case search_type
    when /^r$/ # simple replacement
      if @cfg['validate_links']
        unless valid_link?(search_terms)
          return [false, "Link not valid: #{search_terms}", link_text]
        end
      end
      link_text = search_terms if link_text == ''
      return [search_terms, link_text, link_text]
    when /^@t/ # twitter-ify username
      if search_terms.strip =~ /^@?[0-9a-z_$]+$/i
        url, title = social_handle('t', search_terms)
        link_text = title
      else
        return [false, "#{search_terms} is not a valid Twitter handle", link_text]
      end
    when /^@fb?/ # fb-nify username
      if search_terms.strip =~ /^@?[0-9a-z_]+$/i
        url, title = social_handle('f', search_terms)
        link_text = title
      else
        return [false, "#{search_terms} is not a valid Facebook username", link_text]
      end
    when /^@i/ # fb-nify username
      if search_terms.strip =~ /^@?[0-9a-z_]+$/i
        url, title = social_handle('i', search_terms)
        link_text = title
      else
        return [false, "#{search_terms} is not a valid Instagram username", link_text]
      end
    when /^@l/ # fb-nify username
      if search_terms.strip =~ /^@?[0-9a-z_]+$/i
        url, title = social_handle('l', search_terms)
        link_text = title
      else
        return [false, "#{search_terms} is not a valid LinkedIn username", link_text]
      end
    when /^sp(ell)?$/ # replace with spelling suggestion
      res = spell(search_terms)
      if res
        return [res, res, ""]
      else
        url = false
      end
    when /^hook$/
      url, title = search_hook(search_terms)
    when /^h(([sc])([hb])?)*$/
      str = $1
      types = []
      if str =~ /s([hb]*)/
        if $1.length > 1
          types.push('safari_all')
        elsif $1 == 'h'
          types.push('safari_history')
        elsif $1 == 'b'
          types.push('safari_bookmarks')
        end
      end

      if str =~ /c([hb]*)/
        if $1.length > 1
          types.push('chrome_bookmarks')
          types.push('chrome_history')
        elsif $1 == 'h'
          types.push('chrome_history')
        elsif $1 == 'b'
          types.push('chrome_bookmarks')
        end
      end
      url, title = search_history(search_terms, types)
    when /^a$/
      az_url, title = ddg(%Q{site:amazon.com #{search_terms}})
      url, title = amazon_affiliatize(az_url, @cfg['amazon_partner'])

    when /^g$/ # google lucky search
      url, title = ddg(search_terms)

    when /^pb$/
      url, title = pinboard(search_terms)
    when /^wiki$/
      url, title = wiki(search_terms)

    when /^def$/ # wikipedia/dictionary search
      # title, definition, definition_link, wiki_link = zero_click(search_terms)
      # if search_type == 'def' && definition_link != ''
      #   url = definition_link
      #   title = definition.gsub(/'+/,"'")
      # elsif wiki_link != ''
      #   url = wiki_link
      #   title = "Wikipedia: #{title}"
      # end
      fix = spell(search_terms)

      if fix && search_terms.downcase != fix.downcase
        add_error('Spelling', "Spelling altered for '#{search_terms}' to '#{fix}'")
        search_terms = fix
        link_text = fix
      end

      url, title = define(search_terms)
    when /^imov?$/ #iTunes movie search
      dev = false
      url, title = itunes('movie',search_terms,dev,@cfg['itunes_affiliate'])
    when /^masd?$/ # Mac App Store search (mas = itunes link, masd = developer link)
      dev = search_type =~ /d$/
      url, title = itunes('macSoftware',search_terms, dev, @cfg['itunes_affiliate'])

    when /^itud?$/ # iTunes app search
      dev = search_type =~ /d$/
      url, title = itunes('iPadSoftware',search_terms, dev, @cfg['itunes_affiliate'])

    when /^s$/ # software search (google)
      url, title = ddg(%Q{-site:postmates.com -site:download.cnet.com -site:softpedia.com -site:softonic.com -site:macupdate.com (software OR app OR mac) #{search_terms}})
      unless @titleize
        link_text = title if link_text == ''
      end

    when /^am/ # apple music search
      stype = search_type.downcase.sub(/^am/,'')
      otype = 'link'
      if stype =~ /e$/
        otype = 'embed'
        stype.sub!(/e$/,'')
      end
      case stype
      when /^pod$/
        result = applemusic(search_terms, 'podcast')
      when /^art$/
        result = applemusic(search_terms, 'music', 'musicArtist')
      when /^alb$/
        result = applemusic(search_terms, 'music', 'album')
      when /^song$/
        result = applemusic(search_terms, 'music', 'musicTrack')
      else
        result = applemusic(search_terms)
      end

      # {:type=>,:id=>,:url=>,:title=>}
      if otype == 'embed' && result[:type] =~ /(album|song)/
        url = 'embed'
        if result[:type] =~ /song/
          link = %Q{https://embed.music.apple.com/#{@cfg['country_code']}/album/#{result[:album]}?i=#{result[:id]}&app=music#{@cfg['itunes_affiliate']}}
          height = 150
        else
          link = %Q{https://embed.music.apple.com/#{@cfg['country_code']}/album/#{result[:id]}?app=music#{@cfg['itunes_affiliate']}}
          height = 450
        end

        title = %Q{<iframe src="#{link}" allow="autoplay *; encrypted-media *;" frameborder="0" height="#{height}" style="width:100%;max-width:660px;overflow:hidden;background:transparent;" sandbox="allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation-by-user-activation"></iframe>}
      else
        url = result[:url]
        title = result[:title]
      end

    when /^ipod$/
      url, title = itunes('podcast', search_terms, false)

    when /^isong$/ # iTunes Song Search
      url, title = itunes('song', search_terms, false)

    when /^iart$/ # iTunes Artist Search
      url, title = itunes('musicArtist', search_terms, false)

    when /^ialb$/ # iTunes Album Search
      url, title = itunes('album', search_terms, false)

    when /^lsong$/ # Last.fm Song Search
      url, title = lastfm('track', search_terms)

    when /^lart$/ # Last.fm Artist Search
      url, title = lastfm('artist', search_terms)
    else
      if search_terms
        if search_type =~ /.+?\.\w{2,4}$/
          url, title = ddg(%Q{site:#{search_type} #{search_terms}})
        else
          url, title = ddg(search_terms)
        end
      end
    end
    if link_text == ''
      if @titleize
        link_text = title
      else
        link_text = search_terms
      end
    end

    if url && @cfg['validate_links'] && !valid_link?(url) && search_type !~ /^sp(ell)?/
      [false, "Not found: #{url}", link_text]
    elsif !url
      [false, "No results: #{url}", link_text]
    else
      [url, title, link_text]
    end
  end

end



sl = SearchLink.new({:echo => false})
overwrite = true
backup = sl.cfg['backup']

if ARGV.length > 0
  files = []
  ARGV.each {|arg|
    if arg =~ /^(--?)?(h(elp)?|v(ersion)?)$/
      $stdout.puts "SearchLink v#{VERSION}"
      sl.help_cli
      $stdout.puts "See http://brettterpstra.com/projects/searchlink/ for help"
      Process.exit
    elsif arg =~ /^--?(stdout)$/
      overwrite = false
    elsif arg =~ /^--?no[\-_]backup$/
      backup = false
    else
      files.push(arg)
    end
  }
  files.each {|file|
    if File.exists?(file) && %x{file -b "#{file}"|grep -c text}.to_i > 0
      if RUBY_VERSION.to_f > 1.9
        input = IO.read(file).force_encoding('utf-8')
      else
        input = IO.read(file)
      end
      FileUtils.cp(file,file+".bak") if backup && overwrite

      sl.parse(input)

      if overwrite
        File.open(file, 'w') do |f|
          f.puts sl.output
        end
      else
        puts sl.output
      end
    else
      $stderr.puts "Error reading #{file}"
    end
  }
else
  if RUBY_VERSION.to_f > 1.9
    input = STDIN.read.force_encoding('utf-8').encode
  else
    input = STDIN.read
  end

  sl.parse(input)
  if sl.clipboard
    print input
  else
    print sl.output
  end
end

Thanks for this.

I was looking up Amazon affiliate links and different sources claim different information is required in the affiliate URL. This article makes a claim for much shorter links which only include the affiliate tag.

The google scholar search !gs does not seem to be working correctly. No matter what paper I search for, the result is a 2009 Supreme Court decision.