Agglomerative clustering in Spark

SparkAgglomerativeClustering.java

// sourced from http://users.eecs.northwestern.edu/~cji970/pub/cjinBigDataService2015.pdf

JavaRDD<String> subGraphIdRDD = sc.textFile(idFileLoc,numGraphs);
JavaPairRDD<Integer, Edge> subMSTs = subGraphIdRDD.flatMapToPair(new LocalMST(filesLoc, numSplits));

numGraphs = numSplits * numSplits / 2;
numGraphs = (numGraphs + (K - 1)) / K;

JavaPairRDD<Integer, Iterable<Edge>> mstToBeMerged = subMSTs.combineByKey( new CreateCombiner(), new Merger(),new KruskalReducer(numPoints),numGraphs);
while (numGraphs > 1) {
    numGraphs = (numGraphs + (K - 1)) / K;
    mstToBeMerged = mstToBeMerged.mapToPair(new SetPartitionId(K)).reduceByKey(new KruskalReducer(numPoints),numGraphs);
}