automatcially convert any character encoding to UTF-8, line ending to '\n'

toUTF8.py

import glob
import sys
from concurrent.futures import ProcessPoolExecutor
from itertools import chain

from cchardet import UniversalDetector

detector = UniversalDetector()


def detect(file):
    detector.reset()
    with detector:
        for line in file:
            detector.feed(line)
            if detector.done:
                break

    return detector.result


def main(file):
    try:
        with open(file, 'rb+') as f:
            encoding = detect(f)['encoding']
    except (FileNotFoundError, PermissionError):
        return

    if encoding:
        with open(file, encoding=encoding) as f:
            data = f.read()

        with open(file, 'w', encoding='utf-8', newline='') as f:
            f.write(data)


if __name__ == '__main__':
    files = chain.from_iterable(map(glob.glob, sys.argv[1:]))

    with ProcessPoolExecutor() as executor:
        executor.map(main, files)

Considerably improve performance