run pyspark copy/move from R

runPysparkFromR.R

library(processx)
library(stringr)

runPysparkCopy <- function(fsFrom, fsTo, move=FALSE) {
    pyscript_file <- tempfile(tmpdir="/tmp")
    fileh<-file(pyscript_file)
    copy_or_move <- if(move) "mv" else "cp"
    writeLines(stringr::str_interp("
from pyspark.dbutils import DBUtils
from pyspark.context import SparkContext
from pyspark.sql.session import SparkSession
sc = SparkContext('local')
spark = SparkSession(sc)
print(type(spark))
dbutils = DBUtils(spark)
dbutils.fs.${copy_or_move}('${fsFrom}', '${fsTo}', True)
sc.stop()"), fileh)
    close(fileh)
    log::Info("*********************************** Copying with R. From:", fsFrom, " to: ", fsTo)
    p <- processx::process$new("bash", c("-c", "pyspark"), stdin=pyscript_file,stdout = "|")
    p$read_all_output_lines() #this waits too, p$wait() is also possible
    p$get_exit_status()

}