Parse network traffic with PhantomJS browser and Ruby

network_traffic.rake

# MIT License

# Copyright (c) 2019 Justyna Wojtczak

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

require "capybara/poltergeist"
require 'json'
require 'httparty'

Capybara.register_driver :poltergeist do |app|
  options = {
      js_errors: false,
      debug: false,
      phantomjs_options: ['--load-images=no', '--disk-cache=false'],
      phantomjs_logger: File.open(File::NULL, 'w') #turn off the logs come from console
  }
  Capybara::Poltergeist::Driver.new(app, options)
end
Capybara.always_include_port = false

def cont(r)
  r =~ /cmsEventsInRange/i
end

namespace :network_traffic do
  desc "This task parses the network traffic from given resource url"
  task :parse, [:url] => :environment do |_, arg|
    session = Capybara::Session.new(:poltergeist)
    urls = []
    session.visit arg.url
    session.driver.network_traffic.each do |request|
      next unless cont(request.url)
      urls << request.url
    end
    session.driver.clear_network_traffic

    urls.each do |url|
      response = HTTParty.get(url)
      puts JSON.parse(response.body)
    end
  end
end

# example usage:
# rake network_traffic:parse["https://colonyhs-cjuhsd-ca.schoolloop.com/pf4/cms2_site/view_deployment?d=x&theme_id=i2gg88a1x1oh1oq&group_id=1500178971803"]