README.md

Automated Mapping of Metadata to Concepticon

See the blog at https://calc.hypotheses.org for details.

To run the code, just type:

$ python code.py

code.py

from cldfcatalog import Config
from csvw.dsv import UnicodeDictReader
from collections import defaultdict

repos = Config.from_file().get_clone('concepticon')
paths = {p.stem.split('-')[1]: p for p in
        repos.joinpath('mappings').glob('map-*.tsv')}

mappings = {}
for language, path in paths.items():
    mappings[language] = defaultdict(set)
    with UnicodeDictReader(path, delimiter='\t') as reader:
        for line in reader:
            gloss = line['GLOSS'].split('///')[1]
            mappings[language][gloss].add(
                    (line['ID'], int(line['PRIORITY'])))

esdata = defaultdict(list)
with UnicodeDictReader(
        'spanish-data.tsv', 
        delimiter='\t') as reader:
    for i, line in enumerate(reader):
        # check for direct match
        if line['word'] in mappings['es']:
            best_match, priority = sorted(
                    mappings['es'][line['word']],
                    key=lambda x: x[1])[0]
            esdata[best_match] += [[
                str(i+1),
                line['word'],
                line['averageAoA'],
                best_match,
                priority]]

with open('spanish-data-mapped.tsv', 'w') as f:
    f.write('ID\tWORD\tAOA\tCONCEPTICON_ID\tMATCH\n')
    for key, lines in esdata.items():
        best_line = sorted(
                lines,
                key=lambda x: x[-1])[0]
        best_line[-1] = str(best_line[-1])
        f.write('\t'.join(best_line)+'\n')

print('Found {0} direct matches in data.'.format(
    len(esdata)))

requirements.txt

appdirs==1.4.3
attrs==19.3.0
cdstarcat==1.1.0
certifi==2019.11.28
chardet==3.0.4
cldfbench==1.0.2
cldfcatalog==1.3.0
clldutils==3.5.0
colorlog==4.1.0
csvw==1.6.0
docopt==0.6.2
gitdb2==2.0.6
GitPython==3.0.5
idna==2.8
importlib-metadata==1.4.0
isodate==0.6.0
latexcodec==2.0.0
more-itertools==8.1.0
nameparser==1.0.5
packaging==20.0
pluggy==0.13.1
py==1.8.1
pybtex==0.22.2
pycdstar==1.0.1
pycldf==1.10.0
pyconcepticon==2.5.1
pyparsing==2.4.6
pytest==5.3.4
python-dateutil==2.8.1
PyYAML==5.3
requests==2.22.0
rfc3986==1.3.2
six==1.14.0
smmap2==2.0.5
tabulate==0.8.6
termcolor==1.1.0
Unidecode==1.1.1
uritemplate==3.0.1
urllib3==1.25.7
wcwidth==0.1.8
zipp==2.0.0

spanish-data-mapped.tsv

spanish-data.tsv