tokenizerstring.py

#!/usr/bin/env python3
# coding: utf-8

import os
import pprint
import sys

def tokenize_file(file_name):
    output = []
    with open(file_name, 'r', encoding="utf-8") as f:
        for line in f:
            linetoks = []
            if line.rstrip().startswith("#"):
                output.append([line])
            else:
                i = 0
                start = 0
                while i < len(line):
                    if line[i] == '"':
                        if start < i:
                            linetoks.append(line[start:i])
                        start = i
                        i = i + 1
                        while i < len(line) and (line[i] != '"' or line[i-1] == '\\'):
                            i = i + 1
                        i = i + 1
                        linetoks.append(line[start:i])
                        start = i
                    else:
                        i = i + 1
                if start < i:
                    linetoks.append(line[start:i])
            output.append(linetoks)
    return output

def handle(t):
    if t.startswith('"'):
        return 'std::string(' + t + ')'
    else:
        return t

def main():
    toks = tokenize_file(sys.argv[1])
    for line in toks:
        sys.stdout.write(''.join([handle(x) for x in line]))

if __name__ == '__main__':
    main()