Extract 10,000 useful words from CSV

vocab_extract.r

#install.packages('tm')
library(tm)

#install.packages('slam')
library("slam")

#import data
alldata <- read.csv('stackexchange/20161215StatsPostsMerged.csv', header = TRUE, stringsAsFactors = FALSE)

#make corpus
corp <- Corpus(VectorSource(alldata$CleanBody))

#corpus processing steps
corp <- tm_map(corp, stripWhitespace)

dtm <- DocumentTermMatrix(corp,
           control = list(
             tolower = TRUE,
             stemming = TRUE, 
             stopwords = TRUE,
             minWordLength = 3,
             removeNumbers = TRUE, 
             removePunctuation = TRUE))

#compute term frequency-inverse document frequency (tf-idf)
term_tfidf <- 
  tapply(dtm$v/row_sums(dtm)[dtm$i], dtm$j, mean) *
  log2(nDocs(dtm)/col_sums(dtm > 0))

#omit terms that are very frequent over many documents
dtm <- dtm[,term_tfidf >= median(term_tfidf)]

#omit insignificant terms
dtm <- dtm[row_sums(dtm) > 0,]

#compute term frequency matrix
FreqMat <- data.frame(ST = colnames(dtm), 
                      Freq = col_sums(dtm), 
                      row.names = NULL)
FreqMat <- FreqMat[order(-FreqMat$Freq),]
FreqMat <- FreqMat[!FreqMat$ST %in% stopwords("en"),]

#output most frequent terms from reduced DTM
vocabList <- head(FreqMat, 10000)

#write out to CSV
write.csv(vocabList,file='vocab.csv')