public
Last active

  • Download Gist
rubyget.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
require 'open-uri'
require 'nokogiri'
require 'net/http'
 
# Script tested on 1.9.2-Head Ruby (Linux)
#
# Nokogiri gem install required
 
class RubyGet
 
attr_reader :hostsInfo
 
def initialize(url)
@url = "http://www.microsoft.com"
puts "#{yellowText('initialized script with')} #{blueText(@url)}"
@userAgent = "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4"
@doc = Nokogiri::HTML(open(@url, "User-Agent" => @userAgent))
@hostsInfo = []
@someURLURLs = []
@scriptURLs = []
@cssURLs = []
find_all_elements('img', @someURLURLs)
size_of_element('img', @someURLURLs)
find_all_elements('script', @scriptURLs)
size_of_element('script',@scriptURLs)
find_all_elements('link', @cssURLs)
size_of_element('link',@cssURLs)
puts "#{yellowText("Results found:")}"
@hostsInfo.each {|info| puts "#{info.inspect}"}
puts "Generating output file"
generate_output_file
end
 
# will find all elements in the @doc of name
# * element
# and populate the array
# * array
# with the elements src or href
def find_all_elements(element, array)
# Find all elements with CSS selector
@doc.css(element).each do |link|
# copy src url to array
if(element.eql? "link")
# Check if the <link> tag is a CSS type
if(link['type'].eql? "text/css")
# Check if the <link> href is not nil
if(link['href'] != nil)
# Add the href (not src)
array << link['href']
end
end
else
# Check if the <tag> src is not nil
if(link['src'] != nil)
array << link['src']
end
end
end
end
 
# will try to find all content-lengths in array
# * array
# for the element
# * element
def size_of_element(element, array)
urlThreads = []
totalSize = 0
# Find the content-length of the elements
for someURL in array
urlThreads << Thread.new(someURL) do |urlThread|
host = URI.parse(@url).host
# get path of someURL resourse
path = URI.parse(urlThread).path
# check if url is absolute or relative
if (URI.parse(urlThread).host != nil)
# in this case it's absolute (has a host)
host = URI.parse(urlThread).host
else
# if it's relative, check that a leading / is infront of the path
if (path.index('/') != 0)
path = "/#{path}"
end
end
# print out and add to totalSize
Net::HTTP.start(host) do |http|
query = ""
if(URI.parse(urlThread).query != nil)
query = "?" + URI.parse(urlThread).query
end
response = http.get(path + query)
store_content_length(element, response, host)
end
end
urlThreads.each {|thr| thr.join}
end
end
 
def store_content_length(element, response, host)
if (response.code == "200")
puts "[*] Element #{element} Code =#{greenText(response.code)}#{response.message} for #{host}"
hostFound = @hostsInfo.detect {|info| info[:host].eql? host}
if(hostFound == nil)
# if doesnt contain domain
hostInfo = {
host: host,
images: 0,
scripts: 0,
css: 0
}
@hostsInfo << determine_content_type_and_store(hostInfo, response, element)
else
# if does contain domain
hostFound = determine_content_type_and_store(hostFound, response, element)
end
else
puts "[!!] Element #{element} Code =#{redText(response.code)}#{response.message} for #{host}"
end
end
def generate_output_file
lines = File.readlines('chartTemplate.html')
lines.each_with_index do |line, index|
line.chomp!
if(line.eql? "//0")
lines.insert(index+1,"images = [10,10,10]")
lines.insert(index+2,"scripts = [50,10,40]")
lines.insert(index+3,"css = [0,12,0]")
elsif (line.eql? "//1")
lines.insert(index+1,"var tickers = ['online.vodafone.com', 'statatse.webtrends.com', 'vodafoneuk.122.2o7.net'];")
elsif (line.eql? "//2")
lines.insert(index+1,"plot = $.jqplot('chart1', [images, scripts, css], {")
elsif (line.eql? "//3")
lines.insert(index+1,"yaxis:{ticks:[0, 20, 40, 60, 80]}")
end
end
File.open('chart.html', "w") do |file|
lines.each{|line| file.puts line }
end
end
def determine_content_type_and_store (hostInfo, response, element)
if (element.eql? "img")
hostInfo[:images] = hostInfo[:images] + response['content-length'].to_i
elsif (element.eql? "link")
hostInfo[:css] = hostInfo[:css] + response['content-length'].to_i
elsif (element.eql? "script")
hostInfo[:scripts] = hostInfo[:scripts] + response['content-length'].to_i
end
return hostInfo
end
 
def greenText(text)
"\e[32m #{text} \e[0m"
end
 
def redText(text)
"\e[31m #{text} \e[0m"
end
 
def yellowText(text)
"\e[33m #{text} \e[0m"
end
 
def blueText(text)
"\e[34m #{text} \e[0m"
end
 
end
 
rubyGet = RubyGet.new(ARGV[0])

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.