Script to run spider, saving log output and stuff

runspider.sh

#!/bin/bash

set -e

abort() {
    echo "$*"; exit 1;
}

usage() {
    echo $*
    abort "Usage: $(basename $0) [-h|--help] SPIDER_FILE"
}

require() {
    type $1 >/dev/null 2>/dev/null
}

while [ "${1#-}" != "$1" ]; do
    case "$1" in
        -h|--help) usage;;
        *) usage;;
    esac
    shift
done

require scrapy || abort "Please install scrapy first"
[ -f "$1" ] || abort "Please provide a spider file"

spider="$1"
base=$(basename $1 .py)

output=visited_${spider}-$(date +%Y-%m-%d_%H%M).jl

echo "Running spider, output will be in $output"
(scrapy runspider $spider -o $output 2>&1) > log-$base.txt