sort.sh

directory="./"
save_dir="./SAVED"

##Scan and merge
find "$directory" -type f -name "*.tmp" -print0 | while read -d $'\0' file
do
    named="SortYearMonthDate"
    filename=$(basename "$file")
    year=$(echo "$filename"|cut -d "-" -f 1)
    month=$(echo "$filename"|cut -d "-" -f 2)
    day=$(echo "$filename"|cut -d "-" -f 3| cut -d . -f 1)
    filesave="$save_dir/$named/$year/$month/$year-$month-$day.merged"
    mkdir -p "$save_dir/$named/$year/$month"
    printf "%10s %10s %10s %10s" $file "($(cat $file|wc -l))" "->" $filesave
    cat $file>>$filesave
    printf "%10s\n" "($(cat $filesave|wc -l))"
    # echo -ne "Total lines $(cat $file|wc -l)\n"
done




##SORTING UNIQ
uniq_directory_to_scan="$save_dir"
uniq_extension_to_scan=".merged"

find "$uniq_directory_to_scan" -type f -name "*$uniq_extension_to_scan" -print0 | while read -d $'\0' file
do
    filename=$(basename "$file")
    temp=$(mktemp)
    printf "%10s %80s %10s %10s %10s" "Sort Unique :" $file "->" "($(cat $file|wc -l))" "->";
    cat $file|sort -u > $temp
    cat $temp>$file
    printf "%10s\n" "($(cat $file|wc -l))"
    rm -f $temp
    
done

##Sort TLD
find "$save_dir" -type f -name "*.merged" -print0 | while read -d $'\0' file
do
    named="Domain"
    for domain in $(cat $file | rev | cut -d . -f 1 | rev | sort -u)
    do
        echo "save tld .$domain to $save_dir/$named/$domain.tld"
        mkdir -p "$save_dir/$named"
        cat "$file"|grep -F ".$domain" >> $save_dir/$named/$domain.tld
    done
done


##SORTING UNIQ
uniq_directory_to_scan="$save_dir"
uniq_extension_to_scan=".tld"

find "$uniq_directory_to_scan" -type f -name "*$uniq_extension_to_scan" -print0 | while read -d $'\0' file
do
    filename=$(basename "$file")
    temp=$(mktemp)
    printf "%10s %80s %10s %10s %10s" "Sort Unique :" $file "->" "($(cat $file|wc -l))" "->";
    cat $file|sort -u > $temp
    cat $temp>$file
    printf "%10s\n" "($(cat $file|wc -l))"
    rm -f $temp
    
done