Pinyin_tokenizing_regex

Pinyin_tokenizing_regex.py

pinyin_syllable_regex_accents = """(?:·|‧)?(?:(?:(?:[zcs]h|[gkh])u[aāàáǎ]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[jqx]i[oōóǒò]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[nljqx]i[aāàáǎ]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[dtnlgkhrjqxy])u[aāàáǎ]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h|[gkh])u[aāàáǎ]i)|(?:(?:[zc]h?|[rdtnlgkhsy])[oōóǒò]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhw])?[eēéěè]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?[aāàáǎ]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[bpmdtnljqxy][iīíǐì]ng(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[bpmdtnljqx]i[aāàáǎ]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[bpmdtnljqx]i[aāàáǎ]o)|(?:[nl](?:v|u:|ü)[eēéěè])|(?:[nl](?:[vüǖǘǚǜ]|u:))|(?:[jqxy]u[eēéěè])|(?:[bpmnljqxy][iīíǐì]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:[mdnljqx]i[uūúǔù])|(?:[bpmdtnljqx]i[eēéěè])|(?:[dljqx]i[aāàáǎ])|(?:(?:[zcs]h?|[rdtnlgkhxqjy])[uūúǔù]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[rdtgkh])u[iīíǐì])|(?:(?:[zcs]h?|[rdtnlgkh])u[oōóǒò])|(?:(?:[zcs]h|[rgkh])u[aāàáǎ])|(?:(?:[zcs]h?|[rbpmfdngkhw])?[eēéěè]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?[aāàáǎ]n(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))|(?:(?:[zcs]h?|[rpmfdtnlgkhy])?[oōóǒò]u)|(?:(?:[zcs]h?|[rbpmdtnlgkhy])?[aāàáǎ]o)|(?:(?:[zs]h|[bpmfdtnlgkhwz])?[eēéěè]i)|(?:(?:[zcs]h?|[bpmdtnlgkhw])?[aāàáǎ]i)|(?:(?:[zcs]h?|[rjqxybpmdtnl])[iīíǐì])|(?:(?:[zcs]h?|[rwbpmfdtnlgkhjqxwy])[uūúǔù])|(?:[eēéěè](?:r(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:))?)|(?:(?:[zcs]h?|[rmdtnlgkhy])[eēéěè])|(?:[bpmfwyl]?[oōóǒò])|(?:(?:[zcs]h|[bpmfdtnlgkhzcswy])?[aāàáǎ])|(?:r(?![aāàáǎeēéěèiīíǐìoōóǒòuūúǔùvüǖǘǚǜ]|u:)))[0-5]?"""

pinyin_syllable_regex_numbers_only = """(?:·|‧)?(?:(?:(?:[zcs]h|[gkh])u[a]ng(?![aeiouvü]|u:))|(?:[jqx]i[o]ng(?![aeiouvü]|u:))|(?:[nljqx]i[a]ng(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[dtnlgkhrjqxy])u[a]n(?![aeiouvü]|u:))|(?:(?:[zcs]h|[gkh])u[a]i)|(?:(?:[zc]h?|[rdtnlgkhsy])[o]ng(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhw])?[e]ng(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?[a]ng(?![aeiouvü]|u:))|(?:[bpmdtnljqxy][i]ng(?![aeiouvü]|u:))|(?:[bpmdtnljqx]i[a]n(?![aeiouvü]|u:))|(?:[bpmdtnljqx]i[a]o)|(?:[nl](?:v|u:|ü)[e])|(?:[nl](?:[vü]|u:))|(?:[jqxy]u[e])|(?:[bpmnljqxy][i]n(?![aeiouvü]|u:))|(?:[mdnljqx]i[u])|(?:[bpmdtnljqx]i[e])|(?:[dljqx]i[a])|(?:(?:[zcs]h?|[rdtnlgkhxqjy])[u]n(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[rdtgkh])u[i])|(?:(?:[zcs]h?|[rdtnlgkh])u[o])|(?:(?:[zcs]h|[rgkh])u[a])|(?:(?:[zcs]h?|[rbpmfdngkhw])?[e]n(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[rbpmfdtnlgkhwy])?[a]n(?![aeiouvü]|u:))|(?:(?:[zcs]h?|[rpmfdtnlgkhy])?[o]u)|(?:(?:[zcs]h?|[rbpmdtnlgkhy])?[a]o)|(?:(?:[zs]h|[bpmfdtnlgkhwz])?[e]i)|(?:(?:[zcs]h?|[bpmdtnlgkhw])?[a]i)|(?:(?:[zcs]h?|[rjqxybpmdtnl])[i])|(?:(?:[zcs]h?|[rwbpmfdtnlgkhjqxwy])[u])|(?:[e](?:r(?![aeiouvü]|u:))?)|(?:(?:[zcs]h?|[rmdtnlgkhy])[e])|(?:[bpmfwyl]?[o])|(?:(?:[zcs]h|[bpmfdtnlgkhzcswy])?[a])|(?:r(?![aeiouvü]|u:)))[0-5]?"""