Skip to content

Instantly share code, notes, and snippets.

@tk0miya
Created November 28, 2018 15:52
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tk0miya/275627a00e8d264f5a0e069aabcc7172 to your computer and use it in GitHub Desktop.
Save tk0miya/275627a00e8d264f5a0e069aabcc7172 to your computer and use it in GitHub Desktop.
diff --git a/docutils/docutils/parsers/rst/languages/af.py b/docutils/docutils/parsers/rst/languages/af.py
index 792bfca21..e3335d160 100644
--- a/docutils/docutils/parsers/rst/languages/af.py
+++ b/docutils/docutils/parsers/rst/languages/af.py
@@ -104,3 +104,6 @@ roles = {
'rou': 'raw',}
"""Mapping of Afrikaans role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/ca.py b/docutils/docutils/parsers/rst/languages/ca.py
index 78159ec05..4a807a9fa 100644
--- a/docutils/docutils/parsers/rst/languages/ca.py
+++ b/docutils/docutils/parsers/rst/languages/ca.py
@@ -123,3 +123,6 @@ roles = {
u'cru': 'raw',}
"""Mapping of Catalan role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/cs.py b/docutils/docutils/parsers/rst/languages/cs.py
index 55e037cfa..45e9071da 100644
--- a/docutils/docutils/parsers/rst/languages/cs.py
+++ b/docutils/docutils/parsers/rst/languages/cs.py
@@ -106,3 +106,6 @@ roles = {
u'raw (translation required)': 'raw',}
"""Mapping of Czech role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/da.py b/docutils/docutils/parsers/rst/languages/da.py
index 0e40bcb9f..ca4be60af 100644
--- a/docutils/docutils/parsers/rst/languages/da.py
+++ b/docutils/docutils/parsers/rst/languages/da.py
@@ -111,3 +111,6 @@ roles = {
u'rå': 'raw',}
"""Mapping of Danish role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/de.py b/docutils/docutils/parsers/rst/languages/de.py
index 15168e922..b8e8d4f53 100644
--- a/docutils/docutils/parsers/rst/languages/de.py
+++ b/docutils/docutils/parsers/rst/languages/de.py
@@ -103,3 +103,6 @@ roles = {
u'roh': 'raw',}
"""Mapping of German role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/en.py b/docutils/docutils/parsers/rst/languages/en.py
index 0cbfc78d5..1021678e7 100644
--- a/docutils/docutils/parsers/rst/languages/en.py
+++ b/docutils/docutils/parsers/rst/languages/en.py
@@ -108,3 +108,6 @@ roles = {
'raw': 'raw',}
"""Mapping of English role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/eo.py b/docutils/docutils/parsers/rst/languages/eo.py
index b47240c80..905593969 100644
--- a/docutils/docutils/parsers/rst/languages/eo.py
+++ b/docutils/docutils/parsers/rst/languages/eo.py
@@ -116,3 +116,6 @@ roles = {
}
"""Mapping of Esperanto role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/es.py b/docutils/docutils/parsers/rst/languages/es.py
index 16d3a1c49..3992c35f7 100644
--- a/docutils/docutils/parsers/rst/languages/es.py
+++ b/docutils/docutils/parsers/rst/languages/es.py
@@ -123,3 +123,6 @@ roles = {
}
"""Mapping of Spanish role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/fa.py b/docutils/docutils/parsers/rst/languages/fa.py
index 5547759f5..d24006c64 100644
--- a/docutils/docutils/parsers/rst/languages/fa.py
+++ b/docutils/docutils/parsers/rst/languages/fa.py
@@ -100,3 +100,6 @@ roles = {
u'خام': u'raw',}
"""Mapping of Persian role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/fi.py b/docutils/docutils/parsers/rst/languages/fi.py
index cdc8c33ff..3b5c06924 100644
--- a/docutils/docutils/parsers/rst/languages/fi.py
+++ b/docutils/docutils/parsers/rst/languages/fi.py
@@ -96,3 +96,6 @@ roles = {
u'raw (translation required)': 'raw',}
"""Mapping of Finnish role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/fr.py b/docutils/docutils/parsers/rst/languages/fr.py
index 8328e542a..8eabe358f 100644
--- a/docutils/docutils/parsers/rst/languages/fr.py
+++ b/docutils/docutils/parsers/rst/languages/fr.py
@@ -101,3 +101,6 @@ roles = {
u'brut': 'raw',}
"""Mapping of French role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/gl.py b/docutils/docutils/parsers/rst/languages/gl.py
index 53d52c9c8..3bb40bb71 100644
--- a/docutils/docutils/parsers/rst/languages/gl.py
+++ b/docutils/docutils/parsers/rst/languages/gl.py
@@ -109,3 +109,6 @@ roles = {
u'cru': 'raw',}
"""Mapping of Galician role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/he.py b/docutils/docutils/parsers/rst/languages/he.py
index c23f677e4..0062cc9e8 100644
--- a/docutils/docutils/parsers/rst/languages/he.py
+++ b/docutils/docutils/parsers/rst/languages/he.py
@@ -106,3 +106,6 @@ roles = {
'raw': 'raw',}
"""Mapping of English role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/it.py b/docutils/docutils/parsers/rst/languages/it.py
index 79bf12163..0d4f499e2 100644
--- a/docutils/docutils/parsers/rst/languages/it.py
+++ b/docutils/docutils/parsers/rst/languages/it.py
@@ -95,3 +95,6 @@ roles = {
'grezzo': 'raw',}
"""Mapping of Italian role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/ja.py b/docutils/docutils/parsers/rst/languages/ja.py
index c87bf59ca..a501c7ba1 100644
--- a/docutils/docutils/parsers/rst/languages/ja.py
+++ b/docutils/docutils/parsers/rst/languages/ja.py
@@ -117,3 +117,6 @@ roles = {
u'生': 'raw',}
"""Mapping of Japanese role names to canonical role names for interpreted
text."""
+
+east_asian_widths = 'WFA'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/lt.py b/docutils/docutils/parsers/rst/languages/lt.py
index 49f17dc04..a4e00b402 100644
--- a/docutils/docutils/parsers/rst/languages/lt.py
+++ b/docutils/docutils/parsers/rst/languages/lt.py
@@ -107,3 +107,6 @@ roles = {
'žalia': 'raw',}
"""Mapping of English role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/lv.py b/docutils/docutils/parsers/rst/languages/lv.py
index 307b4b920..8f3158d6e 100644
--- a/docutils/docutils/parsers/rst/languages/lv.py
+++ b/docutils/docutils/parsers/rst/languages/lv.py
@@ -106,3 +106,6 @@ roles = {
'burtiski': 'raw',}
"""Mapping of English role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/nl.py b/docutils/docutils/parsers/rst/languages/nl.py
index 3538e9de2..5213f4765 100644
--- a/docutils/docutils/parsers/rst/languages/nl.py
+++ b/docutils/docutils/parsers/rst/languages/nl.py
@@ -110,3 +110,6 @@ roles = {
'onbewerkt': 'raw',}
"""Mapping of Dutch role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/pl.py b/docutils/docutils/parsers/rst/languages/pl.py
index 9cd7ddbf6..04e440119 100644
--- a/docutils/docutils/parsers/rst/languages/pl.py
+++ b/docutils/docutils/parsers/rst/languages/pl.py
@@ -97,6 +97,6 @@ roles = {
u'surowe': 'raw',}
"""Mapping of Polish role names to canonical role names for interpreted text.
"""
-
-
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/pt_br.py b/docutils/docutils/parsers/rst/languages/pt_br.py
index b0b116759..9cb1b8bf4 100644
--- a/docutils/docutils/parsers/rst/languages/pt_br.py
+++ b/docutils/docutils/parsers/rst/languages/pt_br.py
@@ -106,3 +106,6 @@ roles = {
'cru': 'raw',}
"""Mapping of Brazilian Portuguese role names to canonical role names
for interpreted text."""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/ru.py b/docutils/docutils/parsers/rst/languages/ru.py
index 707e7a58f..45744c0f0 100644
--- a/docutils/docutils/parsers/rst/languages/ru.py
+++ b/docutils/docutils/parsers/rst/languages/ru.py
@@ -87,3 +87,6 @@ roles = {
u'сырой': 'raw',}
"""Mapping of Russian role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/sk.py b/docutils/docutils/parsers/rst/languages/sk.py
index 15a72bf49..87a10bb53 100644
--- a/docutils/docutils/parsers/rst/languages/sk.py
+++ b/docutils/docutils/parsers/rst/languages/sk.py
@@ -93,3 +93,6 @@ roles = {
u'raw (translation required)': 'raw',}
"""Mapping of Slovak role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/sv.py b/docutils/docutils/parsers/rst/languages/sv.py
index 3343fd245..47de35b07 100644
--- a/docutils/docutils/parsers/rst/languages/sv.py
+++ b/docutils/docutils/parsers/rst/languages/sv.py
@@ -93,3 +93,6 @@ roles = {
u'rå': 'raw',}
"""Mapping of Swedish role names to canonical role names for interpreted text.
"""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/zh_cn.py b/docutils/docutils/parsers/rst/languages/zh_cn.py
index 7aa4ea338..3e2bea19b 100644
--- a/docutils/docutils/parsers/rst/languages/zh_cn.py
+++ b/docutils/docutils/parsers/rst/languages/zh_cn.py
@@ -102,3 +102,6 @@ roles = {
u'raw (translation required)': 'raw',}
"""Mapping of Simplified Chinese role names to canonical role names
for interpreted text."""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/languages/zh_tw.py b/docutils/docutils/parsers/rst/languages/zh_tw.py
index 84fe5d79d..8d4ed04d2 100644
--- a/docutils/docutils/parsers/rst/languages/zh_tw.py
+++ b/docutils/docutils/parsers/rst/languages/zh_tw.py
@@ -107,3 +107,6 @@ roles = {
'raw (translation required)': 'raw',}
"""Mapping of Traditional Chinese role names to canonical role names for
interpreted text."""
+
+east_asian_widths = 'WF'
+"""The east_asign_width category string for double-width'ed characters."""
diff --git a/docutils/docutils/parsers/rst/states.py b/docutils/docutils/parsers/rst/states.py
index 08f7dead4..a7817f640 100644
--- a/docutils/docutils/parsers/rst/states.py
+++ b/docutils/docutils/parsers/rst/states.py
@@ -1705,7 +1705,8 @@ class Body(RSTState):
blank_finish = 0
block.disconnect()
# for East Asian chars:
- block.pad_double_width(self.double_width_pad_char)
+ block.pad_double_width(self.double_width_pad_char,
+ self.memo.language.east_asian_widths)
width = len(block[0].strip())
for i in range(len(block)):
block[i] = block[i].strip()
@@ -1771,7 +1772,8 @@ class Body(RSTState):
self.state_machine.next_line(end - start)
block = lines[start:end+1]
# for East Asian chars:
- block.pad_double_width(self.double_width_pad_char)
+ block.pad_double_width(self.double_width_pad_char,
+ self.memo.language.east_asian_widths)
return block, [], end == limit or not lines[end+1].strip()
def malformed_table(self, block, detail='', offset=0):
diff --git a/docutils/docutils/statemachine.py b/docutils/docutils/statemachine.py
index 7e8075252..c8826e0ef 100644
--- a/docutils/docutils/statemachine.py
+++ b/docutils/docutils/statemachine.py
@@ -1447,7 +1447,7 @@ class StringList(ViewList):
block.data = [line[indent:] for line in block.data]
return block
- def pad_double_width(self, pad_char):
+ def pad_double_width(self, pad_char, widths='WF'): # 'W'ide & 'F'ull-width
"""
Pad all double-width characters in self by appending `pad_char` to each.
For East Asian language support.
@@ -1459,7 +1459,7 @@ class StringList(ViewList):
new = []
for char in line:
new.append(char)
- if east_asian_width(char) in 'WF': # 'W'ide & 'F'ull-width
+ if east_asian_width(char) in widths:
new.append(pad_char)
self.data[i] = ''.join(new)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment