Constructing rhyme networks

AnnotatedRhymeData.md

Constructing and analyzing rhyme networks (accompanying data and code)

This little repository offers data and code accompanying the 5th and 6th blog post in the series From rhymes to networks.

The data is taken from the AntRhyme database. For copy-right reasons, all non-rhyming words have been replaced by dummy symbols.

Since the dataset is still being curated for the time being, I would ask you to contact me before using the data, since I have annotated them myself. In any case, you should cite both the blog post which will appear in August, and also the AntRhyme database as well as the RhyAnt annotation tool:

List, Johann-Mattis (2020): AntRhyme: Annothated Rhyme Database. Version 0.1. Jena: Max Planck Institute for the Science of Human History. https://digling.org/rhyant/

List, Johann-Mattis (2020): RhyAnT: A tool for interactive rhyme annotation. Version 0.1. Jena: Max Planck Institue for the Science of Human History. URL: https://digling.org/calc/rhyant/

Dependencies for Python code

• networkx
• tabulate
• lingpy
• python-igraph

Running the code

To run the code for the fifth blogpost, just type:

$ python code-1.py

To inspect the networks, just take the file edges.tsv and load it into Cytoscape.

Output (visible) is:

|:--------------|------:|
| components    |   996 |
| authors       |    72 |
| poems         |   336 |
| stanzas       |  1544 |
| lines         |  8340 |
| rhyme words   |  3950 |
| words rhyming |  5431 |
| words total   | 49797 |

To run the code for the sixth blgopost, just type:

$ python code-2.py

The results are stored in diredges.tsv and dirnodes.tsv and can accordingly be loaded into Cytoscape.

Output is (this can differ, due to the igraph algorithms, since no seed was used):

# 1 Basic statistics
|:--------------|------:|
| components    |   840 |
| authors       |    72 |
| poems         |   336 |
| stanzas       |  1544 |
| lines         |  8340 |
| rhyme words   |  3104 |
| words rhyming |  7707 |
| words total   | 49797 |
| communities   |   931 |

# 2 Promiscuous Rhymes
| rhyme word   |   links |   occurrences |
|:-------------|--------:|--------------:|
| singen       |       6 |             9 |
| wind         |       6 |            12 |
| rein         |       6 |            17 |
| frei         |       6 |            18 |
| haus         |       7 |            45 |
| nacht        |       7 |            40 |
| an           |       7 |            29 |
| auch         |       7 |            16 |
| zieht        |       7 |            11 |
| schon        |       7 |            12 |
| welt         |       7 |            32 |
| sagen        |       8 |            19 |
| leben        |       8 |            39 |
| schein       |       8 |            26 |
| mehr         |       8 |            25 |
| nicht        |       8 |            32 |
| zeit         |       8 |            36 |
| ein          |       9 |            34 |
| bei          |       9 |            36 |
| sein         |      14 |            87 |

# 3 Frequent Edges
| rhyme word A   | rhyme word B   |   occurrences |
|:---------------|:---------------|--------------:|
| sein           | lein           |            10 |
| aus            | haus           |            10 |
| haus           | aus            |             9 |
| triebe         | liebe          |             9 |
| leben          | geben          |             9 |
| geben          | leben          |             9 |
| zeit           | keit           |             9 |
| nein           | sein           |             8 |
| wieder         | lieder         |             7 |
| nur            | tur            |             7 |
| nommen         | kommen         |             7 |
| nacht          | wacht          |             6 |
| glück          | rück           |             6 |
| licht          | nicht          |             6 |
| lein           | sein           |             6 |
| lauf           | auf            |             6 |
| klein          | sein           |             6 |
| schmerz        | herz           |             6 |
| raus           | haus           |             5 |
| nacht          | macht          |             5 |

code-1.py

import sqlite3
import networkx as nx
from collections import defaultdict
from itertools import combinations
from tabulate import tabulate


def strip_chars(word):
    return ''.join([w for w in word.lower() if w not in
    '’;,«».——-./!?():*–“'+"'"])


def parse_line(line, rhymed, poem, stanza):
    for word in line.split():
        if '[' in word:
            rhymes = []
            in_rhyme = False
            rhyme_words = ['']
            for char in word:
                if char == '[':
                    in_rhyme = True
                    rhymes += ['']
                elif char == ']':
                    in_rhyme = False
                elif char == '_':
                    rhyme_words += ['']
                else:
                    if in_rhyme:
                        rhymes[-1] += char
                    else:
                        rhyme_words[-1] += char

            for rhyme, rword in zip(rhymes, rhyme_words):
                if rhyme != '+':
                    rhymed[rhyme] += [
                            (strip_chars(rword), word, poem, stanza)]

G = nx.Graph()


numpo, numsta, numli, wota = 0, 0, 0, 0
authors = defaultdict(int)
with open('rhyme-data.txt') as f:
    rdata = f.read().split('\n\n\n')
    for idx, poem in enumerate(rdata):
        numpo += 1
        stanzas = poem.split('\n\n')
        authors[stanzas[0].split('\n')[1][9:]] += 1
        for sid, stanza in enumerate(stanzas[1:]):
            numsta += 1
            rhymes = defaultdict(list)
            for line in stanza.split('\n'):
                if line.strip() and not line.startswith('#'):
                    parse_line(line, rhymes, idx, sid)
                    wota += len(line.split())
                    numli += 1
            for key, vals in rhymes.items():
                for (rA, wA, pA, sA), (rB, wB, pB, sB) in combinations(vals, r=2):
                    if rA != rB:
                        try:
                            G[rA][rB]['weight'] += 1
                        except:
                            try:
                                G.node[rA]['occ'] += 1
                                G.node[rA]['poem'] += [(pA, sA)]
                                G.node[rA]['words'] += [wA]
                            except:
                                G.add_node(rA, occ=1, poem=[(pA, pB)], words=[wA])
                            try:
                                G.node[rB]['occ'] += 1
                                G.node[rB]['occ'] += [(pB, sB)]
                                G.node[rB]['words'] += [wB]
                            except:
                                G.add_node(rB, occ=1, poem=[(pB, sB)], words=[wB])
                            G.add_edge(rA, rB, weight=1)

with open('edges.tsv', 'w') as f:
    f.write('nodeA\tnodeB\tweight\n')
    for nA, nB, data in G.edges(data=True):
        f.write(nA+'\t'+nB+'\t'+str(data['weight'])+'\n')
with open('nodes.tsv', 'w') as f:
    f.write('node\tfrequency\toccurrences\twords\n')
    for node, data in G.nodes(data=True):
        f.write(node+'\t'+str(data['occ'])+'\t'+'/'.join(
            ['{0}:{1}'.format(a, b) for a, b in data['poem']])+\
                    '\t'+'//'.join(data['words'])+'\n')


comps = list(nx.connected_components(G))
print(len(comps), len(G), numpo, numsta)
numw = sum([data['occ'] for _, data in G.nodes(data=True)])
table = [
        ['components', len(comps)],
        ['authors', len(authors)],
        ['poems', numpo],
        ['stanzas', numsta],
        ['lines', numli],
        ['rhyme words', len(G)],
        ['words rhyming', numw],
        ['words total', wota]
        ]

print(tabulate(table, tablefmt='pipe'))

code-2.py

import networkx as nx
from collections import defaultdict
from itertools import combinations
from tabulate import tabulate
from lingpy.convert.graph import networkx2igraph


def strip_chars(word):
    return ''.join([w for w in word.lower() if w not in
    '_’;,«».——-./!?():*–“"'+"'"])

def parse_line(line, poem, stanza):
    rhymed = []
    for word in line.split():
        if '[' in word:
            rhymes = []
            in_rhyme = False
            rhyme_words = []
            for char in word:
                if char == '[':
                    in_rhyme = True
                    rhymes += ['']
                    rhyme_words += ['']
                elif char == ']':
                    in_rhyme = False
                else:
                    if in_rhyme:
                        rhymes[-1] += char
                    else:
                        if not rhyme_words:
                            pass
                        else:
                            rhyme_words[-1] += char
            nrhymes, nrhyme_words = [], []
            for rhyme, rword in zip(rhymes, rhyme_words):
                if rhyme != '+':
                    nrhymes += [rhyme]
                    nrhyme_words += [rword]
            rhymes = '-'.join(nrhymes)
            rhyme_words = '_'.join([strip_chars(r) for r in nrhyme_words])
            rhymed += [
                    (rhymes, rhyme_words, word, poem, stanza)]
    return rhymed


G = nx.DiGraph()

numpo, numsta, numli, wota = 0, 0, 0, 0
authors = defaultdict(int)
with open('rhyme-data.txt') as f:
    rdata = f.read().split('\n\n\n')
    for idx, poem in enumerate(rdata):
        numpo += 1
        stanzas = poem.split('\n\n')
        authors[stanzas[0].split('\n')[1][9:]] += 1
        for sid, stanza in enumerate(stanzas[1:]):
            numsta += 1
            # iterate line by line and check before 
            queue = []
            for line in stanza.split('\n'):
                if line.strip() and not line.startswith('#'):
                    queue += parse_line(line, idx, sid)
                    wota += len(line.split())
                    numli += 1
            while queue:
                rA, wA, phA, pA, sA = queue.pop(0)
                try:
                    G.node[wA]['occ'] += 1
                    G.node[wA]['poem'] += [pA]
                    G.node[wA]['words'] += [phA]
                except:
                    G.add_node(wA, occ=1, poem=[pA], words=[phA])

                for i, (rB, wB, phB, pB, sB) in enumerate(queue):
                    if rA == rB and wA != wB:
                        try:
                            G[wA][wB]['weight'] += 1
                            G[wA][wB]['poems'] += [pA]
                        except:
                            G.add_edge(wA, wB, weight=1, poems=[pA])
                        break

GI = networkx2igraph(G)
comms = GI.community_infomap(edge_weights='weight')
for i, comm in enumerate(comms):
    for node in comm:
        G.node[GI.vs[node]['Name']]['community'] = i+1

with open('diredges.tsv', 'w') as f:
    f.write('nodeA\tnodeB\tweight\tpoems\n')
    for nA, nB, data in G.edges(data=True):
        f.write(nA+'\t'+nB+'\t'+str(data['weight'])+'\t'+str(len(set(data['poems'])))+'\n')
with open('dirnodes.tsv', 'w') as f:
    f.write('node\tfrequency\toccurrences\twords\tcommunity\n')
    for node, data in G.nodes(data=True):
        f.write(node+'\t'+str(data['occ'])+'\t'+'/'.join(
            ['{0}'.format(a) for a in data['poem']])+\
                    '\t'+'//'.join(data['words'])+'\t'+str(data['community'])+'\n')

comps = list(nx.connected_components(G.to_undirected()))

numw = sum([data['occ'] for _, data in G.nodes(data=True)])
table = [
        ['components', len(comps)],
        ['authors', len(authors)],
        ['poems', numpo],
        ['stanzas', numsta],
        ['lines', numli],
        ['rhyme words', len(G)],
        ['words rhyming', numw],
        ['words total', wota],
        ['communities', len(comms)]
        ]

print('# 1 Basic statistics')
print(tabulate(table, tablefmt='pipe'))
print('')
print('# 2 Promiscuous Rhymes')
table = [[node, len(G[node]), data['occ']] for node, data in
        sorted(G.nodes(data=True), key=lambda x: len(G[x[0]]))[-20:]]
print(tabulate(table, headers=['rhyme word', 'links', 'occurrences'],
    tablefmt='pipe'))

print('')
print('# 3 Frequent Edges')
table = [[nA, nB, len(set(data['poems']))] for nA, nB, data in
        sorted(G.edges(data=True), key=lambda x: len(set(x[2]['poems'])),
            reverse=True)[:20]]
print(tabulate(table, headers=['rhyme word A', 'rhyme word B', 'occurrences'],
    tablefmt='pipe'))pprint(tabulate(table, headers=['rhyme word', 'links', 'occurrences'],
    tablefmt='pipe'))
rint(tabulate(table, headers=['rhyme word', 'links', 'occurrences'],
    tablefmt='pipe'))

diredges.tsv