https://github.com/neubig/nlptutorial

nlp.py

#00
import numpy as np
data,ans=[],[]
with open("00-test-input.txt","r") as inp:
    for line in [s.strip() for s in inp.readlines()]:
        data.append(line.split())
data=sum(data,[])
for word in data:
    ans.append([word,data.count(word)])
for a in np.unique(ans, axis=0):
    print(" ".join(a))
#01-train
import numpy as np
data=[]
ans=[]
with open("01-train-input.txt","r") as inp:
    for line in inp:
        data.append(line.split())
        data.append(['</s>'])
data=sum(data,[])
def one_gram(word,data):
        return [word,'{:.06f}'.format(data.count(word)/len(data))]
for s in np.unique(data):
    ans.append(one_gram(s,data))
for a in np.unique(ans,axis=0):
    print("\t".join(a))
#01-test
import numpy as np
import math
import re
model=[]
data=[]
entropy_data=[]
with open("01-train-answer.txt","r") as m:
    for s in m:
        tmp=re.split(r"\t",s)
        tmp[1]=float(tmp[1])
        model.append(tmp)
with open("01-test-input.txt","r") as d:
    for li in d:
        data.append(re.findall(r"\S",li.strip()))
        data.append(['</s>'])
data=sum(data,[])
def entropy(model):
    return -math.log(0.95*model+0.05*1/1000000,2)

def coverage(model,data):
    word=[x[0] for x in model]
    red=0
    for st in set(data)-set(word):
        red+=data.count(st)
    return (len(data)-red)/len(data)
dic={model[i][0]:model[i][1] for i in range(0,len(model))}
for d in data:
    try:
        entropy_data.append(entropy(dic[d]))
    except KeyError:
        entropy_data.append(entropy(0))
print('entropy = %f'%(sum(entropy_data)/5))import numpy as np
import re
model=[]
data=[]
dict={}
with open("02-train-input.txt","r") as d:
    for line in d.readlines():
        data.append(line.split())
        data.append(['</s>','</s>'])
data=sum(data,[])
data.insert(0,'</s>')
data.pop(-1)
#2-gram
print(data)
for i in range(1,len(data)):
    num=0
    s_word=[data[i-1],data[i]]
    for ii in range(0,len(data)):
        if s_word[0]==data[ii-1] and s_word[1]==data[ii]:
            num+=1
    dict[" ".join(s_word)]=num/data.count(s_word[0])
#     print(num,data.count(s_word[0]))
#1-gram
tmp=data
for i in range(0,data.count('</s>')):
    tmp.remove('</s>')
for w in np.unique(data, axis=0):
    dict[w]=float(data.count(w)/len(tmp))
[print("%s\t%f"%(li[0],li[1])) for li in sorted(dict.items())]
print('coverage = %f'%coverage(model,data))
#02-train(unsolveddddddddd)
import numpy as np
import re
model=[]
data=[]
dict={}
with open("02-train-input.txt","r") as d:
    for line in d.readlines():
        data.append(line.split())
        data.append(['</s>','</s>'])
data=sum(data,[])
data.insert(0,'</s>')
data.pop(-1)
#2-gram
print(data)
for i in range(1,len(data)):
    num=0
    s_word=[data[i-1],data[i]]
    for ii in range(0,len(data)):
        if s_word[0]==data[ii-1] and s_word[1]==data[ii]:
            num+=1
    dict[" ".join(s_word)]=num/data.count(s_word[0])
#     print(num,data.count(s_word[0]))
#1-gram
tmp=data
for i in range(0,data.count('</s>')):
    tmp.remove('</s>')
for w in np.unique(data, axis=0):
    dict[w]=float(data.count(w)/len(tmp))
[print("%s\t%f"%(li[0],li[1])) for li in sorted(dict.items())]
#03