find deal-breakers for 'iconv -f utf8 -t big5'

db-iconv.py

#!/usr/bin/python3
# find deal-breakers for 'iconv -f utf8 -t big5'
# 例： 清冠一號中醫診所清單裡面有一些簡體字「号」「络」和放大鏡符號等等，
# 會造成 iconv 從 utf8 轉 big5 時失敗。 這個程式可以列出有哪幾列有問題。
# https://docs.google.com/spreadsheets/d/e/2PACX-1vQjf_HNeEZKM-XJX-q5v4cfNrB3kcv4gOT8kFbV9rurfoX_H5Qv9112Pv0PgYNFSzbReyNlQkLrJib3/pubhtml#
# 使用方式： python3 db-iconv 某中文utf8編碼檔
# 每列呼叫一次 iconv， 有點慢


import argparse, re
from subprocess import Popen, PIPE, DEVNULL

parser = argparse.ArgumentParser(
    description='find deal-breaker for iconv',
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('file', help='中文文字檔')
args = parser.parse_args()

with open(args.file) as F:
    for i, line in enumerate(F):
        # https://stackoverflow.com/questions/163542/how-do-i-pass-a-string-into-subprocess-popen-using-the-stdin-argument
        iconv = Popen(['iconv', '-f', 'utf8', '-t', 'big5'],
            stdin=PIPE,
            stdout=DEVNULL,
            stderr=PIPE,
            universal_newlines=True
        )
        errmsg = iconv.communicate(input=line)[1]
        if errmsg:
            print(line, end='')
            print('! {:4d} {}'.format(i+1, errmsg), end='')