mrkoll.se scraper

command.sh

SEEDPERSON=$1

rm output.csv
urls=$(curl -s "$SEEDPERSON" | grep /person/ | grep -v Sammanf | grep grannHeader |  sed -r 's/.*href=(.*)>/\1/g' | sed 's/>.*$//' | sed 's/^\//https:\/\/mrkoll.se\//')

#echo "Url, First Name, Last Name, Address, Appt. No., Postcode, DoB, Gender, Res. Since, Phone" >> output.csv
for url in $urls
do
  echo -n "$url," >> output.csv
  curl -s "$url"  | grep -E '(tel:|block_col1)' -A 47 | grep span | sed '/tel:/q'  | sed -E 's/<div class=history_span/\n<div class=history_span/' | grep -vE '(history_span|f_head1|förnamn|f_line1|mellannamn)' | sed 's/<[^>]*>//g' | sed -E 's/ lgh ([0-9]{4})/\nlgh \1\n/' |  sed s/-XXXX// | sed -r '/^\s*$/d' | sed 's/^herre/M/'| sed 's/^tjej/F/'  | sed 's/^dam/F/' | sed 's/^kvinna/F/' | sed 's/^man/M/' | sed '6,7d;' | dos2unix | paste -s -d ',' >> output.csv
done