Jan-Zeiseweis

#!/bin/bash
declare -A arr
for zip_file in *.zip; do
  unzip -q $zip_file && rm $zip_file
  file=${zip_file%.zip}
  read cksm _ < <(md5sum "$file")
  if ((arr[$cksm]++)); then 
    echo "rm $file"
    rm $file
  fi

Jan-Zeiseweis

#!/bin/bash -e

# usage: bash ./split_and_gzip.sh file.txt

FILE=$1
LINES_PER_FILE=10000000 #10m

$(split -l $LINES_PER_FILE -d $FILE ${FILE}.)

FILES=${FILE}.*