Given a list of Twitter handles (one per line), output a list of values for a given profile field

crawl.sh

#!/bin/sh

function usage {
  echo "usage: $0 source_file field"
  exit 1
}

if [ ! $# -eq 2 ]; then
  usage
elif [ ! -f $1 ]; then
  usage
fi


ct=`wc -l $1 | awk -F' ' '{ print $1 }'`
pages=`expr $ct / 100`
if [ `expr $ct % 100` -gt 0 ]; then pages=`expr $pages + 1`; fi

i=0
while [ $i -lt $pages ]; do
  start=`expr $i \* 100 + 1`
  i=`expr $i + 1`
  end=`expr $i \* 100`

  curl -G -d screen_name=`cat $1 | tr "\n" , | cut -d, -f$start-$end` -d include_entities=false http://api.twitter.com/1/users/lookup.json | tr , "\n" | sed -n "/$2/p" | cut -d: -f2
done