Skip to content

Instantly share code, notes, and snippets.

@CerebralMastication
Last active August 11, 2018 14:33
Show Gist options
  • Save CerebralMastication/478b91e4c4f06abc3011cc42fe1e14c9 to your computer and use it in GitHub Desktop.
Save CerebralMastication/478b91e4c4f06abc3011cc42fe1e14c9 to your computer and use it in GitHub Desktop.
Digital Ocean and furrr
# 1. set up digital ocean account
# 2. configure ssh keys, upload public key to DO
# 3. get API key
# 4. set Sys.setenv(DO_PAT = "API_KEY_GOES_HERE") in ~/.Rprofile
#devtools::install_github("sckott/analogsea")
library(analogsea)
library(tidyverse)
library(furrr)
remote_computer <- docklet_create(region = "sfo2", size = "1gb")
# Public IP for droplet(s); this can also be a vector of IP addresses
ip <- remote_computer$networks$v4[[1]]$ip_address
# Path to private SSH key that matches key uploaded to DigitalOcean
ssh_private_key_file <- "/Users/jal/.ssh/id_rsa"
# Connect and create a cluster
cl <- makeClusterPSOCK(
ip,
# User name; DigitalOcean droplets use root by default
user = "root",
# Use private SSH key registered with DigitalOcean
rshopts = c(
"-o", "StrictHostKeyChecking=no",
"-o", "IdentitiesOnly=yes",
"-i", ssh_private_key_file
),
# Command to run on each remote machine
# The script loads the tidyverse Docker image
# --net=host allows it to communicate back to this computer
rscript = c("sudo", "docker", "run", "--net=host",
"rocker/tidyverse", "Rscript"),
# These are additional commands that are run on the remote machine.
# At minimum, the remote machine needs the future library to work—installing furrr also installs future.
rscript_args = c(
# Create directory for package installation
"-e", shQuote("local({p <- Sys.getenv('R_LIBS_USER'); dir.create(p, recursive = TRUE, showWarnings = FALSE); .libPaths(p)})"),
# Install furrr and future
"-e", shQuote("if (!requireNamespace('furrr', quietly = TRUE)) install.packages('furrr')")
),
homogeneous=FALSE,
verbose=TRUE
)
## results in the following:
# > # Connect and create a cluster
# > cl <- makeClusterPSOCK(
# + ip,
# +
# + # User name; DigitalOcean droplets use root by default
# + user = "root",
# +
# + # Use private SSH key registered with DigitalOcean
# + rshopts = c(
# + "-o", "StrictHostKeyChecking=no",
# + "-o", "IdentitiesOnly=yes",
# + "-i", ssh_private_key_file
# + ),
# +
# + # Command to run on each remote machine
# + # The script loads the tidyverse Docker image
# + # --net=host allows it to communicate back to this computer
# + rscript = c("sudo", "docker", "run", "--net=host",
# + "rocker/tidyverse", "Rscript"),
# +
# + # These are additional commands that are run on the remote machine.
# + # At minimum, the remote machine needs the future library to work—installing furrr also installs future.
# + rscript_args = c(
# + # Create directory for package installation
# + "-e", shQuote("local({p <- Sys.getenv('R_LIBS_USER'); dir.create(p, recursive = TRUE, showWarnings = FALSE); .libPaths(p)})"),
# + # Install furrr and future
# + "-e", shQuote("if (!requireNamespace('furrr', quietly = TRUE)) install.packages('furrr')")
# + ),
# +
# + # Actually run this stuff. Set to TRUE if you don't want it to run remotely.
# + dryrun = FALSE,
# + homogeneous=FALSE,
# + verbose=TRUE
# + )
# Workers: [n = 1] ‘206.189.221.35’
# Base port: 11837
# Creating node 1 of 1 ...
# - setting up node
# Starting worker #1 on ‘206.189.221.35’: '/usr/bin/ssh' -R 11837:localhost:11837 -l root -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i /Users/jal/.ssh/id_rsa 206.189.221.35 "'sudo' 'docker' 'run' '--net=host' 'rocker/tidyverse' 'Rscript' --default-packages=datasets,utils,grDevices,graphics,stats,methods -e \"local({p <- Sys.getenv('R_LIBS_USER'); dir.create(p, recursive = TRUE, showWarnings = FALSE); .libPaths(p)})\" -e \"if (!requireNamespace('furrr', quietly = TRUE)) install.packages('furrr')\" -e 'parallel:::.slaveRSOCK()' MASTER=localhost PORT=11837 OUT=/dev/null TIMEOUT=2592000 XDR=TRUE"
# Waiting for worker #1 on ‘206.189.221.35’ to connect back
# Installing package into ‘/usr/local/lib/R/site-library’
# (as ‘lib’ is unspecified)
# also installing the dependencies ‘listenv’, ‘future’, ‘globals’
#
# trying URL 'https://cran.rstudio.com/src/contrib/listenv_0.7.0.tar.gz'
# Content type 'application/x-gzip' length 34981 bytes (34 KB)
# ==================================================
# downloaded 34 KB
#
# trying URL 'https://cran.rstudio.com/src/contrib/future_1.9.0.tar.gz'
# Content type 'application/x-gzip' length 246831 bytes (241 KB)
# ==================================================
# downloaded 241 KB
#
# trying URL 'https://cran.rstudio.com/src/contrib/globals_0.12.1.tar.gz'
# Content type 'application/x-gzip' length 20775 bytes (20 KB)
# ==================================================
# downloaded 20 KB
#
# trying URL 'https://cran.rstudio.com/src/contrib/furrr_0.1.0.tar.gz'
# Content type 'application/x-gzip' length 75030 bytes (73 KB)
# ==================================================
# downloaded 73 KB
#
# * installing *source* package ‘listenv’ ...
# ** package ‘listenv’ successfully unpacked and MD5 sums checked
# ** R
# ** inst
# ** byte-compile and prepare package for lazy loading
# ** help
# *** installing help indices
# ** building package indices
# ** installing vignettes
# ** testing if installed package can be loaded
# * DONE (listenv)
# * installing *source* package ‘globals’ ...
# ** package ‘globals’ successfully unpacked and MD5 sums checked
# ** R
# ** inst
# ** byte-compile and prepare package for lazy loading
# ** help
# *** installing help indices
# ** building package indices
# ** testing if installed package can be loaded
# * DONE (globals)
# * installing *source* package ‘future’ ...
# ** package ‘future’ successfully unpacked and MD5 sums checked
# ** R
# ** demo
# ** inst
# ** byte-compile and prepare package for lazy loading
# ** help
# *** installing help indices
# ** building package indices
# ** installing vignettes
# ** testing if installed package can be loaded
# * DONE (future)
# * installing *source* package ‘furrr’ ...
# ** package ‘furrr’ successfully unpacked and MD5 sums checked
# ** R
# ** byte-compile and prepare package for lazy loading
# ** help
# *** installing help indices
# *** copying figures
# ** building package indices
# ** testing if installed package can be loaded
# * DONE (furrr)
#
# The downloaded source packages are in
# ‘/tmp/RtmpspbvMu/downloaded_packages’
# connect_to localhost: unknown host (nodename nor servname provided, or not known)
############# ^^^ there's my error
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment