欲しい物リストをカーリルで検索 - Find out what books are available on your Amazon wishlist on Calil

.gitignore

config.yml

config.yml.sample

calil:
  appkey: {calil application key}
  systemid: Osaka_Osaka

amazon:
  locale: jp
  email: {your email}

debug: false

wishlist.rb

#!/usr/bin/ruby
# -*- encoding: utf-8 -*-

require 'rubygems'
require 'open-uri'
require 'nokogiri'
require 'net/http'
require 'uri'
require 'kconv'
require 'yaml'


def log(msg)
  p msg
end

String.class_eval do
  def trim
    chomp.strip
  end
end

Object.class_eval do
  def blank?  
    respond_to?(:empty?) ? empty? : !self  
  end  
end

class WishlistItem
  attr_reader :authors, :title, :id

  def initialize (authors, title, id)
    @authors = authors
    @title = title
    @id = id
  end
  def asin
    @id.scan(/^item\.\S+\.(\S+)$/).flatten.join
  end
end

class Wishlist
  attr_reader :numberOfPages, :items, :baseUri, :sid, :gid, :amazonURL, :wishlistID
  PATH_TO_FORM = "//html/body/div/div/div/div/div/div/form"

  def initialize(config)
    @items = Array.new

    # set Amazon URL
    locale = config['locale']
    locale = 'jp' if locale.blank?

    @amazonURL = "http://www.amazon.co.uk"
    if locale == "us"
      @amazonURL = "http://www.amazon.com"
    end
    if locale == "jp"
      @amazonURL = "http://www.amazon.co.jp"
    end
    @config = config
  end
  def fetch(email = nil)

    if email.blank?
      email = @config['email']
    end

    if email.blank?
      raise "Please add your email in config.yml or set --email option"
    end

    # find wishlist ID from email
    log("Fetching wishlist ID from email")

    u = @amazonURL + '/gp/registry/search.html?type=wishlist&field-name=' + email
    url = URI.parse(u)
    full_path = "#{url.path}?#{url.query}"
      the_request = Net::HTTP::Get.new(full_path)


    log("request to " + url.to_s)
    res = Net::HTTP.start(url.host, url.port) { |http|
      http.read_timeout = 10
      http.request(the_request)
    }
    log("... response OK")

    redirect = res['location']
    if redirect == nil
      raise "Cannot find wishlist for #{email} at #{@amazonURL}"
    end

    @wishlistID = redirect.scan(/wishlist\/([a-zA-Z0-9]+)/).flatten.join()
    log("wishlistID = " + @wishlistID)

    # set base URI to retrieve wishlist

    @baseUri = @amazonURL + "/registry/wishlist/" + "#{@wishlistID}"
    # no auth
    # @uriWithParams = @baseUri + "?reveal=unpurchased&filter=3&sort=date-added"
    @compactBaseUri = @baseUri + "?layout=compact&x=15&y=4"

    log("request to " + @compactBaseUri)

    # 'read' is required for reading unicode string
    # http://stackoverflow.com/questions/2572396/nokogiri-open-uri-and-unicode-characters
    doc = Nokogiri::HTML(open(@compactBaseUri).read)
    doc.encoding = 'utf-8'

    setNumberOfPages(doc)

    log("number of pages = " + @numberOfPages.to_s)

    @sid = doc.xpath("//html/body/form/input[@name=\"sid\"]/@value")
    @gid = doc.xpath("//html/body/form/input[@name=\"gid\"]/@value")

    # process each wishlist page

    for pages in 1..@numberOfPages
      if pages > 1
        if debugrun
          break
        end
        @reqUri= @compactBaseUri + "&page=#{pages}"
        log("request to " + @reqUri)
        doc = Nokogiri::HTML(open(@reqUri).read)
      end
      doc.xpath(PATH_TO_FORM + '/table/tbody/tr').each do | row |
        itemid = row.parent['name']
      title = getTitle(row)
      authors = getAuthors(row)
      if title != nil
        if title.trim.length > 0
          @items << WishlistItem.new(authors, title, itemid)
        end
      end
      end
    end

    return @items
  end

  protected

  def debugrun
    @config['debug']
  end

  def setNumberOfPages(doc)
    doc.search("div.list-items div.pagDiv span.pagPage").each do | page_text |
      @numberOfPages = Integer(page_text.inner_text.trim)

    page_number_string = "Page 1 of"
    p = page_text.content
    if p.index(page_number_string) != nil
      @numberOfPages = Integer(p[page_number_string.length, p.length].trim)
      return
    end
    end
  end

  def getAuthors(row)
    authors = Array.new
    row.xpath('td/span[2]').inner_text.toutf8.trim.split(",").each do | author |
      author = author.trim
    if author=~/^by\s/
      author = author[3, author.length]
    end
    if author=~/\(Author\)$/
      author = author[0, author.length - 9]
    end
    authors << author
    end
    return authors
  end

  def getTitle(row)
    title = row.xpath('td/span[1]/strong/a').inner_text.toutf8
    if (title.index("(") != nil)
      title = title[0, title.index("(")]
    end
    colon = title.index(":")
    if colon != nil
      if (title.index(":", colon + 1) != nil)
        title = title[0, title.index(":", colon + 1)]
      end
    end
    if title.index("[") != nil
      title = title[0, title.index("[")]
    end
    return title.chomp
  end
end

class Calil
  def initialize(config)
    @config = config
    raise "Calil appkey required"  if config['appkey'].blank?
    raise "Calil systemid required"  if config['systemid'].blank?
  end
  def debugrun
    @config['debug']
  end
  #
  # items Array [WishlistItem] 
  #
  # return Array [ 
  #   { 
  #     isbn => (String), 
  #     calilurl => (String), 
  #     systems => [
  #       {
  #         systemid
  #         status
  #         libs
  #       }
  #     ], 
  #     oklibs => Integer, 
  #     totallibs => Integer
  #   }
  #  ]
  def request(items)
    asinstr = items.map{ |item| item.asin }.join(',')

    if debugrun
      # テスト用、一件のみ
      asinstr = items.map{ |item| item.asin }.pop
    end

    @reqUri = "http://api.calil.jp/check?appkey=#{@config['appkey']}&isbn=#{asinstr}&systemid=#{@config['systemid']}&format=xml"


    begin
      log("request to #{@reqUri}")
      doc = Nokogiri::HTML(open(@reqUri).read)

      session = doc.xpath('//result/session').inner_text
      continue = doc.xpath('//result/continue').inner_text.to_i

      break if continue == 0

      sleep 3 

      @reqUri = "http://api.calil.jp/check?session=#{session}&format=xml"
    end while continue == 1 # polling

    log("... responose OK")

    return doc.xpath('//result/books/book').map{ |book|
      isbn = book.attr('isbn')
      calilurl =  book.attr('calilurl')
      systems = []
      totallibs = 0
      oklibs = 0
      book.xpath('system').each do |system|
        systems << {
          "systemid" => system.attr('systemid'),
          "status" => system.xpath('status').inner_text,
          "libs" => system.xpath('libkeys/libkey').inject(Hash.new){ |h, libkey| 
          h[libkey.attr('name')] = libkey.inner_text
          totallibs = totallibs + 1
          if libkey.inner_text == '貸出可'
            oklibs = oklibs + 1
          end
          h
        }
        }
      end

      {
        "isbn" => book.attr('isbn'),
        "calilurl" => book.attr('calilurl'),
        "systems" => systems,
        "oklibs" => oklibs,
        "totallibs" => totallibs
      }
    }
  end

  def search(items)

    # 数値以外で開始しているitemは除外(書籍ではない？)
    items.reject! { |item| /^[a-zA-Z]/ =~ item.asin }

    # TODO: fileter unique item
    itemHash = items.inject(Hash.new(0)) {|h, item| h[item.asin] = item; h}

    # 一度のリクエストで取得可能な最大書籍数は 100
    n = 50 

    result = []
    while items.size > 0 do
      log(sprintf("%d / %d", itemHash.size - items.size, itemHash.size))

      reqitems = items.slice!(0, n);
      result.concat(request(reqitems))
    end 


    result.sort{|a, b|
      b['oklibs'] <=> a['oklibs'] # 降順
    }.each{|d|
      puts sprintf("%s %2d / %2d %s", d['isbn'], d['oklibs'], d['totallibs'], itemHash[d['isbn']].title)
    }
  end
end


configPath = File.dirname(File.expand_path(__FILE__)) + "/config.yml"

if !FileTest.exists?(configPath)
  puts "Please make a config.yml"
  exit
end

puts "Load from #{configPath}"
config = YAML.load_file(configPath)
config = Hash.new{ |hash,key| hash[key] = {} } if config == false
config['amazon'] = Hash.new{ |hash,key| hash[key] = {} } if config['amazon'].blank?
config['calil'] = Hash.new{ |hash,key| hash[key] = {} } if config['calil'].blank?

require 'optparse'
opt = OptionParser.new
opt.on('--appkey   = Calil Application Key') {|v| config['calil']['appkey'] = v }
opt.on('--systemid = Calil systemid') {|v| config['calil']['systemid'] = v }
opt.on('--email    = your amazon email') {|v,| config['amazon']['email'] = v }
opt.on('--locale   = amazon locale') {|v| config['amazon']['locale'] = v }

opt.parse!

# override debug option
if config['debug']
  config['amazon']['debug'] = config['debug']
  config['calil']['debug'] = config['debug']
end

p config

wishlist = Wishlist.new(config['amazon'])

Calil.new(config['calil']).search(wishlist.fetch(config['amazon']['email']))