This gist demonstrates how to perform unsupervised hierarchical clustering

R hierarchical clustering

library(RColorBrewer)
library(gplots)

# Generate some random data
N_SAMPLES=20
N_GENES=30
x<- matrix(data = rnorm(600), nrow = N_GENES, ncol = N_SAMPLES)
rownames(x) <- paste("genes",1:N_GENES)
colnames(x) <- paste("sample",1:N_SAMPLES)
head(x)

cl<-as.dist(1-cor(t(x), method="spearman"))
hr <- hclust(cl , method="complete")
# need to optimise the cluster size - depends on your project
mycl <- cutree(hr, h=max(hr$height/1.3))
clusterCols <- brewer.pal(length(unique(mycl)),"Paired")
myClusterSideBar <- clusterCols[mycl]
colfunc <- colorRampPalette(c("blue", "white", "red"))
write.table(mycl,file="GeneClusters1.txt",quote=F,sep="\t")

# create heatmap
heatmap.2(x, main="Gene Clustering 1",  Rowv=as.dendrogram(hr),
          dendrogram="both", scale="none", col = colfunc(25), trace="none", 
          RowSideColors= myClusterSideBar, margins = c(5,5))