Skip to content

Instantly share code, notes, and snippets.

@ibombonato
Created March 1, 2019 18:32
Show Gist options
  • Save ibombonato/9a2934c05bb8da23fc7e20aaf2613872 to your computer and use it in GitHub Desktop.
Save ibombonato/9a2934c05bb8da23fc7e20aaf2613872 to your computer and use it in GitHub Desktop.
R run sql query in parallel with foreach, DBI and odbc packages
library(foreach)
library(doParallel)
library(DBI)
#Creating the cluster
cl <- makeCluster(detectCores() -1)
# Defining packages and variables for cluster
clusterEvalQ(cl, {
library(odbc)
library(DBI)
conn <- dbConnect(odbc::odbc(),
.connection_string = "Driver={SQL Server};Server=MyServer;Database=MyDatabase;UID=MyUser;PWD=MyPwd;")
NULL
})
#Register the Cluster
registerDoParallel(cl)
#Start the Parallel Loop
results <- foreach(i= 1:10, .combine = "rbind", .inorder = FALSE) %dopar% {
sql_text <- "SELECT * FROM MyTable WHERE ID = ?id"
sql_query <- DBI::sqlInterpolate(conn, sql_text, id = i)
q <- DBI::dbGetQuery(conn, sql_query)
}
#Closing connection in all clusters
clusterEvalQ(cl, {
dbDisconnect(conn)
})
# Stopping cluster
stopCluster(cl)
stopImplicitCluster()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment