cluster learning

cluster.kt

package wekaTest

import weka.clusterers.AbstractClusterer
import weka.clusterers.FilteredClusterer
import weka.clusterers.SimpleKMeans
import weka.core.EuclideanDistance
import weka.core.Instances
import weka.core.SelectedTag
import weka.core.converters.ConverterUtils.DataSource
import weka.filters.unsupervised.attribute.Remove

fun main() {
    val data = DataSource.read("data/identify.csv")
    // 拿出200个作为测试
    val testSet = data.takeOutTestSet(200.toDouble() / data.numInstances())
    val km = kmeans(data)
    test(testSet, km)
}

fun kmeans(data: Instances): AbstractClusterer {
    val fc = FilteredClusterer()
    println("[Kmeans] initializing...")
    val km = SimpleKMeans()
    // 簇数
    km.numClusters = 50
    // Kmeans++
    km.initializationMethod = SelectedTag(SimpleKMeans.KMEANS_PLUS_PLUS, SimpleKMeans.TAGS_SELECTION)
    // 距离函数
    km.distanceFunction = EuclideanDistance()
    // 线程数
    km.numExecutionSlots = 8

    fc.clusterer = km
    // 移除前两列：index和label
    fc.filter = Remove().also { it.attributeIndices = "first-2" }
    println("[Kmeans] clustering...")
    fc.buildClusterer(data)
    println("[Kmeans] cluster finished. squared error: ${km.squaredError}")
    return fc
}

fun test(testSet: Instances, clusterer: AbstractClusterer) {
    //            testSet, clusterer
    var a = 0  // same, same
    var b = 0  // diff, same
    var c = 0  // same, diff
    var d = 0  // diff, diff

    val n = testSet.numInstances()
    println("[Test] running. test set size: $n")
    for (i in 0 until n) {
        for (j in i until n) {
//            println("testing $i $j ...")
            val ins1 = testSet[i]
            val ins2 = testSet[j]
            val testSame = ins1.value(1) ==
                    ins2.value(1)
            val clustererSame = clusterer.clusterInstance(ins1) ==
                    clusterer.clusterInstance(ins2)
            if (testSame && clustererSame) {
//                println("same, same")
                a++
            } else if (!testSame && clustererSame) {
//                println("different, same")
                b++
            } else if (testSame && !clustererSame) {
//                println("same, different")
                c++
            } else {
//                println("different, different")
                d++
            }
        }
    }
//    println("$a, $b, $c, $d")
    println("[Test] finished. jaccard index: ${(a + d).toDouble() / (a + b + c + d)}")
}

fun Instances.takeOutTestSet(rate: Double): Instances {
    val testSetNum = (rate * this.numInstances()).toInt()
    val testSet = Instances(this, 0, testSetNum)
    this.forEachIndexed { idx, _ ->
        if (idx < testSetNum)
            this.removeAt(idx)
    }
    return testSet
}