Lamar Smith

gistfile1.sh

#!/usr/bin/env bash

set -x

DEST=data
NOW=`date '+%s'`
WGET_WARC=./wget-warc
USER_AGENT="Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2) Gecko/20070225 lolifox/0.32"

BASE="$DEST/texansforlamarsmith-${NOW}"

mkdir -p tmp
mkdir -p $DEST

$WGET_WARC \
        -U "${USER_AGENT}" \
        -e "robots=off" \
        -nv \
        -r \
        -l inf \
        -o "$BASE.log" \
        --directory-prefix="tmp" \
        --warc-file="$BASE" \
        --no-remove-listing \
        --warc-header="operator: Archive Team" \
        --no-timestamping \
        --page-requisites --trust-server-names \
        "http://www.texansforlamarsmith.com/"