bookmarkive filter script

filter.sh

#!/bin/sh
# take input from stdin and sort, remove duplicates and filter

set -e
cat /dev/stdin | sort | uniq | \

  # strip hash endings (will kill twitter)
  sed -e "s/#.*//" | \
  
  # remove session id strings
  sed -e "s/&sid=.*$//" | \
  
# #   remove terminal /; but why?
#   sed -e "s/\/$//" | \
  
# #   adhoc removal of urls based on keywords; -i is --ignore-case, -v is --invert-match
#   grep -i -v -e 4chan -e facebook -e watchseries -e videoweed -e tesco -e dropbox -e flickr -e block\.opendns
  grep -i -v -e "^javascript" -e "^file" -e "^about" -e "^http\:\/\/192\." -e "\.swf" -e "\.flv" | \
  
  # pass through uniq again as stripping endings may have produced multiples
  uniq