hsiboy

#!/bin/bash

#
# Takes IIS W3C formatted log files, and creates a load file for use with siege.
# 
#


# make sure you always put $f in double quotes to avoid any nasty surprises i.e. "$f"
for f in $@

hsiboy

logfile = siege-result.csv
verbose = true
timeout = 122
show-logfile = false
logging = true
protocol = HTTP/1.1
chunked = true
cache = false
connection = close
concurrent = 15

hsiboy

#!/bin/bash

# make sure you always put $f in double quotes to avoid any nasty surprises i.e. "$f"
for f in $@
do

if [ ! -f $f ];
then
    echo "File ($f) not found!"
    exit 0

hsiboy

www.about.com akadns.net
www.adobe.com
www.amazon.com
www.amazon.co.uk
www.aol.com akadns.net
www.aol.co.uk akadns.net
www.apple.com akadns.net
www.argos.co.uk
www.asda.co.uk
www.ask.com edgesuite.net

hsiboy

#!/bin/bash

NOWD=$(date +"%Y-%m-%d")

URLS="
http://www.domain.com/page1.html
http://www.domain.com/page2.html
http://www.domain.com/pagen.html
http://www.domain.com/and/so/on.html
"

hsiboy

# Automatically limit all requests from any referer.
# Uses two rate classes, BusyReferer for those sites that send a large amount of traffic
# and StandardReferers for those that don't.

# Referer whitelist. These referers are never rate limited.
$whitelist = "localhost 172.16.121.100";
 
# Referers that are allowed to pass a higher number of clients.
$highTraffic = "google mypartner.com";
 

hsiboy

# implement the following policy:
 
# if transactions complete within 50 ms, do not attempt to shape traffic.
# if transactions take more than 50 ms, assume that we are in danger of overload and Rate-limit traffic to 100 requests per second, and if requests exceed that rate limit, send back a '204 No Content' message rather then queuing them.
# Once transaction time comes down to less than 50ms, remove the rate limit.


# Create a rate shaping class named "Searchlimit" with a max_rate_per_second: 100
# Create a Service Level Monitoring class named SearchTimer with a response_time of 500 ms.

hsiboy

Bot-Buster™

Tracks nefarious activity on website, and manages accordingly.

It's probably a bot.

If the requesting entity:

• declares its user-agent as being wget, curl, webcopier etc - it's probably a bot.
• requests details -> details -> details -> details ad nauseum - it's probably a bot.
• requests the html, but not .css, .js or site furniture - it's probably a bot.

hsiboy

xargs -n1 -P 10 curl -o /dev/null --silent --head --write-out '%{url_effective};%{http_code};%{time_total};%{time_namelookup};%{time_connect};%{size_download};%{speed_download}\n' < awk -F ',' '{print $0}' External_Links.csv

hsiboy

#!/bin/bash
echo -n "url to measure: "
read url
URL="$url"
echo -n "How many times to fetch? "
read loop
NOWD=$(date +"%Y-%m-%d")
clean_var()
{
        INVAR=$1