Created
September 24, 2021 16:33
-
-
Save melardev/3eef393780737b200e765f70ad592ae4 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import argparse | |
from typing import List | |
# https://stackoverflow.com/questions/31306951/how-to-sort-a-list-by-last-character-of-string | |
def last_letter(word): | |
return word[::-1] | |
def run(file_paths: List[str], output_path: str, sort=False,reverse=False, quiet=False): | |
if sort: | |
# set is a lot faster, also it auto sorts | |
unique_lines = set() | |
else: | |
unique_lines = [] | |
for file_path in file_paths: | |
with open(file_path, encoding='utf-8') as fd: | |
for line in fd: | |
line_s = line.rstrip() | |
if line_s not in unique_lines: | |
if sort: | |
unique_lines.add(line_s) | |
else: | |
unique_lines.append(line_s) | |
else: | |
if not quiet: | |
print('Found duplicate ', line) | |
if sort: | |
unique_lines = sorted(unique_lines) | |
elif reverse: | |
unique_lines = sorted(unique_lines, key=last_letter) | |
with open(output_path, 'w', encoding='utf-8') as fd: | |
for unique_line in unique_lines: | |
fd.write(unique_line) | |
fd.write('\n') | |
if __name__ == '__main__': | |
parser = argparse.ArgumentParser() | |
parser.add_argument('-s', '--sort', default=False, | |
required=False, action='store_true', help='sort file') | |
parser.add_argument('-r', '--reverse', default=False, | |
required=False, action='store_true', help='sort file') | |
parser.add_argument('file', type=str, nargs='+', | |
help='The file to work on') | |
args = parser.parse_args() | |
files = args.file | |
should_sort = args.sort | |
reverse = args.reverse | |
print('Files: %s\nSort: %s' % (files, args.sort)) | |
run(files, files[0], sort=should_sort,reverse=reverse, quiet=False) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment