thm1118/test_distance_of_vectorize.py

## test_distance_of_vectorize.py
# -*- coding: utf-8 -*-
"""
    使用sklearn 提供的距离函数测试,对 'euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
                  'braycurtis', 'canberra', 'chebyshev', 'correlation',
                  'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
                  'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
                  'russellrao', 'seuclidean', 'sokalmichener',
                  'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"
        pearson 等20几个度量距离算法进行逐一测试。
        向量空间模型主要使用tfidf
    测试结果：
    braycurtis，chebyshev，correlation，余玄夹角，dice，sokalsneath ，sqeuclidean，皮尔逊 除了能判定雷同外，还能明显排除完全不同。
    jaccard---只能判断完全雷同。欧式和曼哈顿等虽然能判定雷同，但不能明显隔开完全不同。其他算法就只有相似度，无法判定雷同
"""
from sklearn.metrics import pairwise
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
import jieba
import numpy as np
import time

print __doc__
jieba_tokenizer = lambda s: list(jieba.cut(s))
corpus = [
    u'Writing II: Rhetorical Composing',
    u'Genetics and Society: A Course for Educators',
    u'General Game Playing',
    u'Genes and the Human Condition (From Behavior to Biotechnology)',
    u'A Brief History of Humankind',
    u'New Models of Business in Society',
    u'Analyse Numrique pour Ingnieurs',
    u'Evolution: A Course for Educators',
    u'Coding the Matrix: Linear Algebra through Computer Science Applications',
    u'The Dynamic Earth: A Course for Educators',
    u'Tiny Wings\tYou have always dreamed of flying - but your wings are tiny. Luckily the world is full of beautiful hills. Use the hills as jumps - slide down, flap your wings and fly! At least for a moment - until this annoying gravity brings you back down to earth. But the next hill is waiting for you already. Watch out for the night and fly as fast as you can. ',
    u'Angry Birds Free',
    u'没有\它很相似',
    u'没有\t它很相似',
    u'没有\t他很相似',
    u'没有\t他不很相似',
    u'没有',
    u'可以没有',
    u'也没有',
    u'有没有也不管',
    u'Angry Birds Stella',
    u'Flappy Wings - FREE\tFly into freedom!A parody of the #1 smash hit game!',
    u'没有一个',
    u'没有一个2',
]


tfidvectorizer = TfidfVectorizer(tokenizer=jieba_tokenizer)
X = tfidvectorizer.fit_transform(corpus)

print X.shape, u'\n 特征词表', np.reshape(tfidvectorizer.get_feature_names(), (10, 12)), u'\n 向量空间模型', X.toarray()

test = [u'没有']
test_vec = tfidvectorizer.transform(test)
print u'\n 测试：', test_vec.shape, u'\n test向量', test_vec.toarray()


_VALID_METRICS = ['euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
                  'braycurtis', 'canberra', 'chebyshev', 'correlation',
                  'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
                  'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
                  'russellrao', 'seuclidean', 'sokalmichener',
                  'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"]
# 开始比较
''' sklearn自己实现了支持稀疏矩阵 三种度量：欧几里得、曼哈顿、余弦。另外，支持这三个算法的别名l2,l1,cityblock 。
    其余度量是sklearn调用scipy，不支持稀疏矩阵，是基于稠密矩阵的
'''
metrics_for_sparse = ['euclidean', 'manhattan', 'cosine', 'l2', 'l1', 'cityblock']
start = time.time()
#for metric in pairwise.distance_metrics().keys():
for metric in pairwise._VALID_METRICS:
    # 这两个算法报错
    if metric in ['mahalanobis', 'wminkowski']: continue
    if metric in metrics_for_sparse:
        mx = X
        y = test_vec
    else:
        mx = X.todense()
        y = test_vec.todense()

    print u'\n ==========使用距离算法：', metric, "==========="
    D = pairwise.pairwise_distances(mx, y, metric=metric)

    dlist = [(i, d[0]) for i, d in enumerate(D)]
    dlist.sort(key=lambda item: item[1])

    for i, d in dlist:
        print u'距离：', d, u'   ', corpus[i]

print u'\n 消耗时间：', time.time()-start

'''
 使用皮尔逊相关系数，这时候不一定使用tfidf，因为皮尔逊解决就是样本数据落差过大问题。自己会寻找中间拟合线
'''
from scipy.stats import pearsonr

countvectorizer = CountVectorizer(tokenizer=jieba_tokenizer)
X = countvectorizer.fit_transform(corpus)
print X.shape, u'\n 特征词表', np.reshape(tfidvectorizer.get_feature_names(), (10, 12)), u'\n 向量空间模型', X.toarray()
test_vec = countvectorizer.transform(test)
print u'\n 测试：', test_vec.shape, u'\n test向量', test_vec.toarray()
dicts = {}
for i, row in enumerate(X):
    rvalue = pearsonr(row.toarray().ravel(), test_vec.toarray().ravel())
    dicts[i] = rvalue


dlist = [(i, dicts[i][0], dicts[i][1]) for i in dicts.keys()]
dlist.sort(key=lambda item: item[1], reverse=True)
print u'\n========使用皮尔逊相关系数 ======================'
for i, oddsratio, pvalue in dlist:
    print u'让步率：', oddsratio, u'   ', pvalue, ' ', corpus[i]

print

''' 对矩阵内的行进行两两距离计算，返回 相关系数（n_samples,n_samples矩阵）'''
from pylab import *
from pprint import pprint

def arrayToList(arr):
    if type(arr) == type(array([])):
        return arrayToList(arr.tolist())
    elif type([]) == type(arr):
        return [arrayToList(a) for a in arr]
    else:
        return '{:.2f}'.format(arr)


def prettyArray(arr):
    pprint(arrayToList(arr), width=800)

D = pairwise.pairwise_distances(X)
np.set_printoptions(precision=3)
prettyArray(D)
	# -- coding: utf-8 --
	"""
	使用sklearn 提供的距离函数测试,对 'euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
	'braycurtis', 'canberra', 'chebyshev', 'correlation',
	'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
	'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
	'russellrao', 'seuclidean', 'sokalmichener',
	'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"
	pearson 等20几个度量距离算法进行逐一测试。
	向量空间模型主要使用tfidf
	测试结果：
	braycurtis，chebyshev，correlation，余玄夹角，dice，sokalsneath ，sqeuclidean，皮尔逊除了能判定雷同外，还能明显排除完全不同。
	jaccard---只能判断完全雷同。欧式和曼哈顿等虽然能判定雷同，但不能明显隔开完全不同。其他算法就只有相似度，无法判定雷同
	"""
	from sklearn.metrics import pairwise
	from sklearn.feature_extraction.text import TfidfVectorizer
	from sklearn.feature_extraction.text import CountVectorizer
	import jieba
	import numpy as np
	import time

	print __doc__
	jieba_tokenizer = lambda s: list(jieba.cut(s))
	corpus = [
	u'Writing II: Rhetorical Composing',
	u'Genetics and Society: A Course for Educators',
	u'General Game Playing',
	u'Genes and the Human Condition (From Behavior to Biotechnology)',
	u'A Brief History of Humankind',
	u'New Models of Business in Society',
	u'Analyse Numrique pour Ingnieurs',
	u'Evolution: A Course for Educators',
	u'Coding the Matrix: Linear Algebra through Computer Science Applications',
	u'The Dynamic Earth: A Course for Educators',
	u'Tiny Wings\tYou have always dreamed of flying - but your wings are tiny. Luckily the world is full of beautiful hills. Use the hills as jumps - slide down, flap your wings and fly! At least for a moment - until this annoying gravity brings you back down to earth. But the next hill is waiting for you already. Watch out for the night and fly as fast as you can. ',
	u'Angry Birds Free',
	u'没有\它很相似',
	u'没有\t它很相似',
	u'没有\t他很相似',
	u'没有\t他不很相似',
	u'没有',
	u'可以没有',
	u'也没有',
	u'有没有也不管',
	u'Angry Birds Stella',
	u'Flappy Wings - FREE\tFly into freedom!A parody of the #1 smash hit game!',
	u'没有一个',
	u'没有一个2',
	]


	tfidvectorizer = TfidfVectorizer(tokenizer=jieba_tokenizer)
	X = tfidvectorizer.fit_transform(corpus)

	print X.shape, u'\n 特征词表', np.reshape(tfidvectorizer.get_feature_names(), (10, 12)), u'\n 向量空间模型', X.toarray()

	test = [u'没有']
	test_vec = tfidvectorizer.transform(test)
	print u'\n 测试：', test_vec.shape, u'\n test向量', test_vec.toarray()


	_VALID_METRICS = ['euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
	'braycurtis', 'canberra', 'chebyshev', 'correlation',
	'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
	'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
	'russellrao', 'seuclidean', 'sokalmichener',
	'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"]
	# 开始比较
	''' sklearn自己实现了支持稀疏矩阵三种度量：欧几里得、曼哈顿、余弦。另外，支持这三个算法的别名l2,l1,cityblock 。
	其余度量是sklearn调用scipy，不支持稀疏矩阵，是基于稠密矩阵的
	'''
	metrics_for_sparse = ['euclidean', 'manhattan', 'cosine', 'l2', 'l1', 'cityblock']
	start = time.time()
	#for metric in pairwise.distance_metrics().keys():
	for metric in pairwise._VALID_METRICS:
	# 这两个算法报错
	if metric in ['mahalanobis', 'wminkowski']: continue
	if metric in metrics_for_sparse:
	mx = X
	y = test_vec
	else:
	mx = X.todense()
	y = test_vec.todense()

	print u'\n ==========使用距离算法：', metric, "==========="
	D = pairwise.pairwise_distances(mx, y, metric=metric)

	dlist = [(i, d[0]) for i, d in enumerate(D)]
	dlist.sort(key=lambda item: item[1])

	for i, d in dlist:
	print u'距离：', d, u' ', corpus[i]

	print u'\n 消耗时间：', time.time()-start

	'''
	使用皮尔逊相关系数，这时候不一定使用tfidf，因为皮尔逊解决就是样本数据落差过大问题。自己会寻找中间拟合线
	'''
	from scipy.stats import pearsonr

	countvectorizer = CountVectorizer(tokenizer=jieba_tokenizer)
	X = countvectorizer.fit_transform(corpus)
	print X.shape, u'\n 特征词表', np.reshape(tfidvectorizer.get_feature_names(), (10, 12)), u'\n 向量空间模型', X.toarray()
	test_vec = countvectorizer.transform(test)
	print u'\n 测试：', test_vec.shape, u'\n test向量', test_vec.toarray()
	dicts = {}
	for i, row in enumerate(X):
	rvalue = pearsonr(row.toarray().ravel(), test_vec.toarray().ravel())
	dicts[i] = rvalue


	dlist = [(i, dicts[i][0], dicts[i][1]) for i in dicts.keys()]
	dlist.sort(key=lambda item: item[1], reverse=True)
	print u'\n========使用皮尔逊相关系数 ======================'
	for i, oddsratio, pvalue in dlist:
	print u'让步率：', oddsratio, u' ', pvalue, ' ', corpus[i]

	print

	''' 对矩阵内的行进行两两距离计算，返回相关系数（n_samples,n_samples矩阵）'''
	from pylab import *
	from pprint import pprint

	def arrayToList(arr):
	if type(arr) == type(array([])):
	return arrayToList(arr.tolist())
	elif type([]) == type(arr):
	return [arrayToList(a) for a in arr]
	else:
	return '{:.2f}'.format(arr)


	def prettyArray(arr):
	pprint(arrayToList(arr), width=800)

	D = pairwise.pairwise_distances(X)
	np.set_printoptions(precision=3)
	prettyArray(D)