This cleans up CSVs by removing duplicates based on a column, removes all double-quoted commas, and all doublequotes

clean-csv.sh

#!/bin/bash
# This cleans up CSVs

echo "Removing all commas between quotes" #Quick and dirty
echo "Then removing quotes"
echo "Lastly, we remove duplicates based on Column 7 - Phone number"

cat $1 | sed ':a;s/^\(\([^"]*,\?\|"[^",]*",\?\)*"[^",]*\),/\1 /;ta' | \
sed 's/\"//g' | \
sort -t ',' -k 7,7 -u > $1-OUTPUT.csv