This gist contains the complete code for my blogpost: 'Bayesian Machine Learning and NLP with R and sparklyr'

bayes_w_r_and_sparklyr.R

mc$defaultLibrary <- "sparklyr"

library(sparklyr)
library(tidyverse)

speeches <- magpie::sql(mc, "SELECT * FROM presidential_speeches WHERE president")

partitions <- speeches %>%
    ft_tokenizer(input_col = 'speech_text', output_col = 'words') %>%
    ft_stop_words_remover(input_col = 'words', output_col = 'clean_words') %>%
    ft_hashing_tf(input_col = 'clean_words', output_col = 'words_vector', num_features = 2^12) %>%
    sdf_random_split(training = 0.7, test = 0.3, seed = 411)

nb_model <- partitions$training %>%
    ml_naive_bayes(president ~ words_vector)

pred <- ml_predict(nb_model, partitions$test)

ml_multiclass_classification_evaluator(pred)