Tool chain to create shadow configs from the most popular sites according to Alexa

download.rb

# ruby download.rb <hosts-file>
IO.foreach(ARGV[0]) do |l|
  name = l[0...-1] # Remove \n
  Dir.mkdir(name)
  spawn('wget', '-H', '-p', "http://#{name}", :chdir => name)
end

Process.waitall

hosts.erb

<!-- Needed plug-ins -->
<plugin id="filex" path="libshadow-plugin-filetransfer.so" />
<plugin id="browser" path="libshadow-plugin-browser.so" />

<!-- Servers -->

<% servers.each_with_index do |server, i| -%>
<software id="<%= name %>-<%= i %>" plugin="filex" time="10" arguments="server 80 <%= Dir.pwd %>/<%= name %>/<%= server[:name] %>/" />
<node id="<%= server[:name] %>" software="<%= name %>-<%= i %>" cluster="<%= server[:name] %>" bandwidthdown="60000" bandwidthup="30000" cpufrequency="2800000" />
<% end -%>

<!-- Browser -->
<software id="client" plugin="browser" time="20" arguments="<%= name %> 80 none 0 6 /index.html" />
<node id="client.node" software="client" cluster="browser" logpcap="true" quantity="1" />

<!-- Max simulation time -->
<kill time="600" />

list.rb

# ruby list.rb <name-for-hosts-file>
File.open(ARGV[0], 'w') do |f| # Write hosts file
  IO.readlines("top-1m.csv")[0...ARGV[1].to_i].each do |l|
    f.write l.split(",").last
  end
end

topology.erb

<topology>
  <cluster id="browser" bandwidthdown="6000" bandwidthup="1000" />
<% servers.each do |s| -%>
  <cluster id="<%= s[:name] %>" bandwidthdown="100000" bandwidthup="100000" />
<% end -%>
<% servers.each do |s| -%>
  <link id="link-<%= s[:name] %>-browser" clusters="<%= s[:name] %> browser" latency="<%= s[:latency] %>" jitter="<%= s[:jitter] %>" packetloss="<%= s[:loss] %>" />
  <link id="link-browser-<%= s[:name] %>" clusters="browser <%= s[:name] %>" latency="<%= s[:latency] %>" jitter="<%= s[:jitter] %>" packetloss="<%= s[:loss] %>" />
<% end -%>
<% servers.product(servers).each do |s1, s2| -%>
  <link id="link-<%= s1[:name] %>-<%= s2[:name] %>" clusters="<%= s1[:name] %> <%= s2[:name] %>" latency="1" jitter="0" />
<% end -%>
</topology>

xml.rb

%w{erb statsample work_queue net/ping/tcp ostruct}.each { |l| require l }

@templates = {
  :hosts => ERB.new(File.read(File.dirname(__FILE__) + "/hosts.erb"), nil, '-'),
  :topology => ERB.new(File.read(File.dirname(__FILE__) + "/topology.erb"), nil, '-')
}

def config_with_template(name, type, servers)
  namespace = OpenStruct.new(:servers => servers)
  File.open("#{name}/#{type}.xml", 'w') do |f| # Write hosts file
    f.write @templates[type].result(namespace.instance_eval { binding })
  end
end

print "hostname".ljust(32)
print "latency"
print "jitter".rjust(10)
print "loss\n".rjust(10)

wq = WorkQueue.new 15, nil

Dir["*/"].map { |n| n[0...-1] }.each do |name|
  wq.enqueue_b do
    servers = []

    Dir["#{name}/*/"].map { |n| n.split('/').last }.each do |s|
      ping = Net::Ping::TCP.new(s, 'http')

      latencies = (1..10).inject([]) do |l|
        if ping.ping? 
          l << (ping.duration * 1000).round
        else 
          l
        end
      end.to_scale

      loss = (10 - latencies.size) / 10.0
      median = latencies.median 
      q1 = latencies.percentil(25)
      q3 = latencies.percentil(75)
      jitter = ((median - q1) + (q3 - median)) / 2.0
      jitter = [ median - q1, q3 - median].min if jitter >= median

      servers << { 
        :name => s,
        :latency => median.round,
        :jitter => jitter.round,
        :loss => loss
      }
    end
    
    msg = servers.inject("") do |msg, s|
      msg += s[:name].ljust(32)
      msg += "#{s[:latency].to_s.rjust(5)}ms"
      msg += s[:jitter].to_s.rjust(10)
      msg +  "#{(s[:loss] * 100).to_s.rjust(8)}%\n"
    end

    print msg

    config_with_template(name, :hosts, servers)
    config_with_template(name, :topology, servers) 
  end
end

wq.join