uniqfy_file.py

#!/usr/bin/env python3
import argparse
from typing import List

# https://stackoverflow.com/questions/31306951/how-to-sort-a-list-by-last-character-of-string
def last_letter(word):
    return word[::-1]

def run(file_paths: List[str], output_path: str, sort=False,reverse=False, quiet=False):
    if sort:
        # set is a lot faster, also it auto sorts
        unique_lines = set()
    else:
        unique_lines = []

    for file_path in file_paths:
        with open(file_path, encoding='utf-8') as fd:
            for line in fd:
                line_s = line.rstrip()
                if line_s not in unique_lines:
                    if sort:
                        unique_lines.add(line_s)
                    else:
                        unique_lines.append(line_s)
                else:
                    if not quiet:
                        print('Found duplicate ', line)

    if sort:
        unique_lines = sorted(unique_lines)
    elif reverse:
        unique_lines = sorted(unique_lines, key=last_letter)
        
    with open(output_path, 'w', encoding='utf-8') as fd:
        for unique_line in unique_lines:
            fd.write(unique_line)
            fd.write('\n')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--sort', default=False,
                        required=False, action='store_true', help='sort file')
    parser.add_argument('-r', '--reverse', default=False,
                    required=False, action='store_true', help='sort file')
    parser.add_argument('file', type=str, nargs='+',
                        help='The file to work on')

    args = parser.parse_args()

    files = args.file
    should_sort = args.sort
    reverse = args.reverse
    print('Files: %s\nSort: %s' % (files, args.sort))
    run(files, files[0], sort=should_sort,reverse=reverse, quiet=False)