crawling

crawler.rb

#!/usr/bin/ruby

require 'capybara'
require 'capybara/dsl'
require 'capybara/poltergeist'
require 'nokogiri'
require 'open-uri'


Capybara.configure do |config|
    config.run_server = false
    config.current_driver = :poltergeist
    config.javascript_driver = :poltergeist
    config.app_host = "https://console.ng.bluemix.net/"
    config.default_max_wait_time = 60
    config.ignore_hidden_elements = false
end

Capybara.register_driver :poltergeist do |app|
    Capybara::Poltergeist::Driver.new(app, {:timeout=>120, js_errors: false})
end


module Crawler
  class Bluemix

    include Capybara::DSL

    def login(username, password)
      page.driver.headers = { "User-Agent" => "Mac Safari" }

      visit('')

      # <a href="/login" ～>ログイン</a>
      click_on "ログイン"

      fill_in "username", :with => username
      fill_in "password", :with => password

      # <a href="#" ～>log in</a>
      click_on "log in"
    end


    def select(region, org)
      # 地域と組織の管理画面へ遷移
      # 右上顔マークのclickトリガーでajax発生
      find("li.profileSetting").trigger('click')

      # 地域用画面へ遷移
      # 該当の下矢印のclickトリガーでajax発生
      find("img.toggle-region-expand").trigger('click')
      find_all("p.select-name").each do |name|
        if name.text == region
          puts "clicked #{name.text}"
          name.trigger('click')
        end
      end

      # 組織用画面へ遷移
      # 該当の下矢印のclickトリガーでajax発生
      find("img.toggle-org-expand").trigger('click')
      find_all("p.select-name").each do |name|
        if name.text == org
          puts "clicked #{name.text}"
          name.trigger('click')
        end
      end
    end


    def report
      # 課金情報を閲覧できるアカウントリンクをクリック
      find("a.account-link").trigger('click')

      # サービスの右矢印をすべてクリックして情報開示
      find_all("span.i-arrow-right-dark-blue").each do |name|
          puts "clicked blue #{name.text}"
          name.trigger('click')
      end

      # サービスの配下に出てきたインスタンスの右矢印をすべてクリックして情報開示
      find_all("span.i-arrow-right-white").each do |name|
          puts "clicked white #{name.text}"
          name.trigger('click')
      end

      # HTMLの描画を待つ
      # find系コマンドが続く場合はjquery応答中は描画を待ってくれる
      wait_for_ajax
      sleep 30
        
      #html = open(Capybara.app_host)
      #doc = Nokogiri::HTML.parse(html)
      #puts doc
      puts doc.xpath("//div[@class='chargesDetailsServicesContainer hideHeaderLabels']")

      page.save_screenshot('screenshot.png', :full => true)
    end


    def wait_for_ajax
      sleep 2
      Timeout.timeout(Capybara.default_max_wait_time) do
        active = page.evaluate_script('jQuery.active')
        until active == 0 || active == nil
          sleep 1
          active = page.evaluate_script('jQuery.active')
          end
        end
      end

    end
end

crawler = Crawler::Bluemix.new
crawler.login("ユーザネーム", "パスワード")
crawler.select("米国南部", "組織名")
crawler.report