Last active
May 31, 2016 20:01
-
-
Save grigory93/060b1788f498275f86d311b0e0dac26f to your computer and use it in GitHub Desktop.
Running parallel jobs on Aster with R and toaster
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
library(toaster) | |
close(conn) | |
conn = odbcDriverConnect(connection="driver={Aster ODBC Driver};server=10.xx.xx.xx;port=2406;database=dallas;uid=beehive;pwd=beehive", | |
interpretDot=TRUE) | |
dallasPermitsTableInfo = getTableSummary(conn, "dallasbuildingpermits") | |
getNumericColumns(dallasPermitsTableInfo) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fit.folds = foreach(fold = 1:12, .packages=c('RODBC','toaster')) %dopar% { | |
parConn = odbcReConnect(conn) | |
fitted.model = computeLm(parConn, "dallasbuildingpermits", value ~ area + lon + lat, tableInfo = dallasPermitsTableInfo, | |
where = paste("date_part('month',issued)::int != ", fold)) | |
close(parConn) | |
fitted.model | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
library(doParallel) | |
cl = makeCluster(6) | |
registerDoParallel(cl, cores=6) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
library(foreach) | |
# running cross-validation sequantially with foreach function | |
fit.folds = foreach(fold = 1:12, .packages=c('RODBC', 'toaster')) %do% { | |
fitted.model = computeLm(conn, "dallasbuildingpermits", value ~ area + lon + lat, tableInfo = dallasPermitsTableInfo, | |
where = paste("date_part('month',issued)::int != ", fold)) | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fit.folds = list() | |
for(fold in 1:12) { | |
fit.folds[[fold]] = computeLm(conn, "dallasbuildingpermits", value ~ area + lon + lat, tableInfo = dallasPermitsTableInfo, | |
where = paste("date_part('month',issued)::int !=", fold)) | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
install.packages("toaster", "parallel", "iterators", "doParallel", "foreach") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
SELECT DATE_PART('month',issued)::int fold, COUNT(*) fold_size | |
FROM public.dallasbuildingpermits | |
GROUP BY 1 | |
ORDER BY 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
SELECT COUNT(*) | |
FROM public.dallasbuildingpermits | |
WHERE DATE_PART('month',issued)::int != 6 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
value ~ area + lon + lat |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fit.all = computeLm(conn, "dallasbuildingpermits", value ~ area + lon + lat, tableInfo = dallasPermitsTableInfo) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment