Conda environment for R parcel

micro_benchmark.R

> data <- data.frame(y = runif(10^6, 0, 1))
> microbenchmark::microbenchmark(
+   setup = library(arrow),
+   arrow_on = {
+     sparklyr_df <<- copy_to(sc, data, overwrite = T)
+     count(sparklyr_df) %>% collect()
+   },
+   arrow_off = {
+     if ("arrow" %in% .packages()) detach("package:arrow")
+     sparklyr_df <<- copy_to(sc, data, overwrite = T)
+     count(sparklyr_df) %>% collect()
+   },
+   times = 10
+ )  %>% print()

Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table


Attaching package: ‘arrow’

The following object is masked from ‘package:utils’:

    timestamp

The following objects are masked from ‘package:base’:

    array, table

Unit: milliseconds
      expr       min        lq      mean    median       uq      max neval
  arrow_on  789.5403  806.0901  963.8207  926.3281 1097.237 1181.066    10
 arrow_off 5201.8576 5420.6246 5930.0022 5738.2361 6602.012 6934.848    10

R_env.frozen.yml

name: R_env
channels:
  - conda-forge
  - defaults
dependencies:
  - arrow-cpp=0.12.0=py37h0e61e49_0
  - attrs=18.2.0=py_0
  - backcall=0.1.0=py_0
  - binutils_impl_linux-64=2.31.1=h6176602_1
  - binutils_linux-64=2.31.1=h6176602_3
  - bleach=3.1.0=py_0
  - boost-cpp=1.68.0=h11c811c_1000
  - bzip2=1.0.6=h14c3975_1002
  - ca-certificates=2018.11.29=ha4d7672_0
  - cairo=1.14.12=h80bd089_1005
  - certifi=2018.11.29=py37_1000
  - curl=7.63.0=h646f8bb_1000
  - decorator=4.3.2=py_0
  - entrypoints=0.3=py37_1000
  - fontconfig=2.13.1=h2176d3f_1000
  - freetype=2.9.1=h94bbf69_1005
  - gcc_impl_linux-64=7.3.0=habb00fd_1
  - gcc_linux-64=7.3.0=h553295d_3
  - gettext=0.19.8.1=h9745a5d_1001
  - gfortran_impl_linux-64=7.3.0=hdf63c60_1
  - gfortran_linux-64=7.3.0=h553295d_3
  - glib=2.56.2=had28632_1001
  - gmp=6.1.2=hf484d3e_1000
  - graphite2=1.3.13=hf484d3e_1000
  - gxx_impl_linux-64=7.3.0=hdf63c60_1
  - gxx_linux-64=7.3.0=h553295d_3
  - harfbuzz=1.9.0=he243708_1001
  - icu=58.2=hf484d3e_1000
  - ipykernel=5.1.0=py37h24bf2e0_1002
  - ipython=7.2.0=py37h24bf2e0_1000
  - ipython_genutils=0.2.0=py_1
  - jedi=0.13.2=py37_1000
  - jinja2=2.10=py_1
  - jpeg=9c=h14c3975_1001
  - jsonschema=3.0.0a3=py37_1000
  - jupyter_client=5.2.4=py_1
  - jupyter_core=4.4.0=py_0
  - krb5=1.16.3=hc83ff2d_1000
  - libcurl=7.63.0=h01ee5af_1000
  - libedit=3.1.20170329=hf8c457e_1001
  - libffi=3.2.1=hf484d3e_1005
  - libgcc-ng=7.3.0=hdf63c60_0
  - libiconv=1.15=h14c3975_1004
  - libpng=1.6.36=h84994c4_1000
  - libprotobuf=3.6.1=hdbcaa40_1000
  - libsodium=1.0.16=h14c3975_1001
  - libssh2=1.8.0=h1ad7b7a_1003
  - libstdcxx-ng=7.3.0=hdf63c60_0
  - libtiff=4.0.10=h648cc4a_1001
  - libuuid=2.32.1=h14c3975_1000
  - libxcb=1.13=h14c3975_1002
  - libxml2=2.9.8=h143f9aa_1005
  - make=4.2.1=h14c3975_2004
  - markupsafe=1.1.0=py37h14c3975_1000
  - mistune=0.8.4=py37h14c3975_1000
  - mkl_fft=1.0.10=py37h14c3975_1
  - mkl_random=1.0.2=py37h637b7d7_2
  - nbconvert=5.3.1=py_1
  - nbformat=4.4.0=py_1
  - ncurses=6.1=hf484d3e_1002
  - notebook=5.7.4=py37_1000
  - openssl=1.0.2p=h14c3975_1002
  - pandoc=2.5=1
  - pandocfilters=1.4.2=py_1
  - pango=1.40.14=hf0c64fd_1003
  - parso=0.3.2=py_0
  - pcre=8.41=hf484d3e_1003
  - pexpect=4.6.0=py37_1000
  - pickleshare=0.7.5=py37_1000
  - pip=19.0.1=py37_0
  - pixman=0.34.0=h14c3975_1003
  - prometheus_client=0.5.0=py_0
  - prompt_toolkit=2.0.8=py_0
  - pthread-stubs=0.4=h14c3975_1001
  - ptyprocess=0.6.0=py37_1000
  - pygments=2.3.1=py_0
  - pyrsistent=0.14.9=py37h14c3975_1000
  - python=3.7.1=hd21baee_1000
  - python-dateutil=2.7.5=py_0
  - pyzmq=17.1.2=py37h6afc9c9_1001
  - r-assertthat=0.2.0=r351h6115d3f_1001
  - r-backports=1.1.3=r351h96ca727_1000
  - r-base=3.5.1=he45234b_1005
  - r-base64enc=0.1_3=r351h96ca727_1002
  - r-bh=1.69.0_1=r351h6115d3f_0
  - r-bindr=0.1.1=r351h6115d3f_1001
  - r-bindrcpp=0.2.2=r351h29659fb_1001
  - r-boot=1.3_20=r351_1000
  - r-broom=0.5.1=r351h6115d3f_1000
  - r-callr=3.1.1=r351h6115d3f_1000
  - r-caret=6.0_81=r351h96ca727_1000
  - r-cellranger=1.1.0=r351h6115d3f_1001
  - r-class=7.3_15=r351h96ca727_1000
  - r-cli=1.0.1=r351h6115d3f_1000
  - r-clipr=0.5.0=r351h6115d3f_0
  - r-cluster=2.0.7_1=r351ha65eedd_1000
  - r-codetools=0.2_16=r351h6115d3f_1000
  - r-colorspace=1.4_0=r351h96ca727_0
  - r-crayon=1.3.4=r351h6115d3f_1001
  - r-curl=3.3=r351h96ca727_0
  - r-data.table=1.12.0=r351h96ca727_0
  - r-dbi=1.0.0=r351h6115d3f_1001
  - r-dbplyr=1.3.0=r351h6115d3f_1000
  - r-digest=0.6.18=r351h96ca727_1000
  - r-dplyr=0.7.8=r351h29659fb_1000
  - r-essentials=3.5.1=r351_2000
  - r-evaluate=0.12=r351h6115d3f_1000
  - r-fansi=0.4.0=r351h96ca727_1000
  - r-forcats=0.3.0=r351h6115d3f_1001
  - r-foreach=1.4.4=r351h6115d3f_1001
  - r-foreign=0.8_71=r351h96ca727_1002
  - r-formatr=1.5=r351h6115d3f_1001
  - r-fs=1.2.6=r351h29659fb_1000
  - r-generics=0.0.2=r351h6115d3f_1001
  - r-ggplot2=3.1.0=r351h6115d3f_1000
  - r-gistr=0.4.2=r351h6115d3f_1001
  - r-git2r=0.24.0=r351h47c54a8_0
  - r-glmnet=2.0_16=r351ha65eedd_1001
  - r-glue=1.3.0=r351h14c3975_1002
  - r-gower=0.1.2=r351h96ca727_1002
  - r-gtable=0.2.0=r351h6115d3f_1001
  - r-haven=2.0.0=r351h29659fb_1000
  - r-hexbin=1.27.2=r351ha65eedd_1002
  - r-highr=0.7=r351h6115d3f_1001
  - r-hms=0.4.2=r351h6115d3f_1000
  - r-htmltools=0.3.6=r351hf484d3e_1002
  - r-htmlwidgets=1.3=r351h6115d3f_1000
  - r-httpuv=1.4.5.1=r351hf484d3e_1000
  - r-httr=1.4.0=r351h6115d3f_1000
  - r-ipred=0.9_8=r351h96ca727_1000
  - r-irdisplay=0.7=r351_1000
  - r-irkernel=0.8.15=r351h6115d3f_1001
  - r-iterators=1.0.10=r351h6115d3f_1001
  - r-jsonlite=1.6=r351h96ca727_1000
  - r-kernsmooth=2.23_15=r351ha65eedd_1002
  - r-knitr=1.21=r351h6115d3f_1000
  - r-labeling=0.3=r351h6115d3f_1001
  - r-later=0.7.5=r351h29659fb_1000
  - r-lattice=0.20_38=r351h96ca727_1000
  - r-lava=1.6.4=r351h6115d3f_1000
  - r-lazyeval=0.2.1=r351h96ca727_1002
  - r-lubridate=1.7.4=r351h29659fb_1001
  - r-magrittr=1.5=r351h6115d3f_1001
  - r-maps=3.3.0=r351h96ca727_1002
  - r-markdown=0.9=r351h96ca727_1000
  - r-mass=7.3_51.1=r351h96ca727_1000
  - r-matrix=1.2_15=r351h96ca727_1000
  - r-mgcv=1.8_26=r351h96ca727_1000
  - r-mime=0.6=r351h96ca727_1000
  - r-modelmetrics=1.1.0=r351h29659fb_1002
  - r-modelr=0.1.2=r351h6115d3f_1001
  - r-munsell=0.5.0=r351h6115d3f_1001
  - r-nlme=3.1_137=r351ha65eedd_1000
  - r-nnet=7.3_12=r351h96ca727_1002
  - r-numderiv=2016.8_1=r351h6115d3f_1001
  - r-openssl=1.1=r351hff1dc39_1000
  - r-pbdzmq=0.3_3=r351h193a840_1000
  - r-pillar=1.3.1=r351h6115d3f_1000
  - r-pkgconfig=2.0.2=r351h6115d3f_1001
  - r-plogr=0.2.0=r351h6115d3f_1001
  - r-plyr=1.8.4=r351h29659fb_1002
  - r-prettyunits=1.0.2=r351h6115d3f_1001
  - r-processx=3.2.1=r351h96ca727_1000
  - r-prodlim=2018.04.18=r351h29659fb_1002
  - r-progress=1.2.0=r351h6115d3f_1002
  - r-promises=1.0.1=r351h29659fb_1000
  - r-pryr=0.1.4=r351h29659fb_1002
  - r-ps=1.3.0=r351h96ca727_1000
  - r-purrr=0.2.5=r351h96ca727_1002
  - r-quantmod=0.4_13=r351h6115d3f_1000
  - r-r.methodss3=1.7.1=r351h6115d3f_1001
  - r-r.oo=1.22.0=r351h6115d3f_1000
  - r-r.utils=2.7.0=r351h6115d3f_1000
  - r-r6=2.3.0=r351h6115d3f_1000
  - r-randomforest=4.6_14=r351ha65eedd_1000
  - r-rbokeh=0.5.0=r351h6115d3f_1001
  - r-rcolorbrewer=1.1_2=r351h6115d3f_1001
  - r-rcpp=1.0.0=r351h29659fb_1000
  - r-rcpproll=0.3.0=r351h29659fb_1000
  - r-readr=1.3.1=r351h29659fb_1000
  - r-readxl=1.2.0=r351h29659fb_1000
  - r-recipes=0.1.4=r351h6115d3f_1000
  - r-recommended=3.5.1=r351_1001
  - r-rematch=1.0.1=r351h6115d3f_1001
  - r-repr=0.19.1=r351h6115d3f_1000
  - r-reprex=0.2.1=r351h6115d3f_1000
  - r-reshape2=1.4.3=r351h29659fb_1003
  - r-rlang=0.3.1=r351h96ca727_0
  - r-rmarkdown=1.11=r351h6115d3f_1000
  - r-rpart=4.1_13=r351h96ca727_1002
  - r-rprojroot=1.3_2=r351h6115d3f_1001
  - r-rstudioapi=0.9.0=r351h6115d3f_0
  - r-rvest=0.3.2=r351h6115d3f_1001
  - r-scales=1.0.0=r351h29659fb_1001
  - r-selectr=0.4_1=r351h6115d3f_1000
  - r-shiny=1.2.0=r351_1000
  - r-sourcetools=0.1.7=r351hf484d3e_1000
  - r-spatial=7.3_11=r351h96ca727_1002
  - r-squarem=2017.10_1=r351h6115d3f_1001
  - r-stringi=1.2.4=r351h29659fb_1001
  - r-stringr=1.3.1=r351h6115d3f_1001
  - r-survival=2.43_3=r351h96ca727_1000
  - r-tibble=2.0.1=r351h96ca727_0
  - r-tidyr=0.8.2=r351h29659fb_1002
  - r-tidyselect=0.2.5=r351h29659fb_1000
  - r-tidyverse=1.2.1=r351h6115d3f_1001
  - r-timedate=3043.102=r351h6115d3f_1000
  - r-tinytex=0.10=r351h6115d3f_0
  - r-ttr=0.23_4=r351ha65eedd_1000
  - r-utf8=1.1.4=r351h96ca727_1000
  - r-uuid=0.1_2=r351h96ca727_1001
  - r-viridislite=0.3.0=r351h6115d3f_1001
  - r-whisker=0.3_2=r351h6115d3f_1001
  - r-withr=2.1.2=r351h6115d3f_1000
  - r-xfun=0.4=r351h6115d3f_1000
  - r-xml2=1.2.0=r351h29659fb_1002
  - r-xtable=1.8_3=r351_2000
  - r-xts=0.11_1=r351h96ca727_1000
  - r-yaml=2.2.0=r351h96ca727_1001
  - r-zoo=1.8_4=r351h96ca727_1000
  - readline=7.0=hf8c457e_1001
  - send2trash=1.5.0=py_0
  - setuptools=40.7.1=py37_0
  - six=1.12.0=py37_1000
  - sqlite=3.26.0=h67949de_1000
  - terminado=0.8.1=py37_1001
  - testpath=0.4.2=py37_1000
  - thrift-cpp=0.12.0=h23e226f_1000
  - tk=8.6.9=h84994c4_1000
  - tornado=5.1.1=py37h14c3975_1000
  - traitlets=4.3.2=py37_1000
  - wcwidth=0.1.7=py_1
  - webencodings=0.5.1=py_1
  - wheel=0.32.3=py37_0
  - xorg-kbproto=1.0.7=h14c3975_1002
  - xorg-libice=1.0.9=h14c3975_1004
  - xorg-libsm=1.2.3=h4937e3b_1000
  - xorg-libx11=1.6.6=h14c3975_1000
  - xorg-libxau=1.0.8=h14c3975_1006
  - xorg-libxdmcp=1.1.2=h14c3975_1007
  - xorg-libxext=1.3.3=h14c3975_1004
  - xorg-libxrender=0.9.10=h14c3975_1002
  - xorg-renderproto=0.11.1=h14c3975_1002
  - xorg-xextproto=7.3.0=h14c3975_1002
  - xorg-xproto=7.0.31=h14c3975_1007
  - xz=5.2.4=h14c3975_1001
  - zeromq=4.2.5=hf484d3e_1006
  - zlib=1.2.11=h14c3975_1004
  - _r-mutex=1.0.0=anacondar_1
  - blas=1.0=mkl
  - bwidget=1.9.11=1
  - intel-openmp=2019.1=144
  - libgfortran-ng=7.3.0=hdf63c60_0
  - mkl=2019.1=144
  - numpy=1.15.4=py37h7e9f1db_0
  - numpy-base=1.15.4=py37hde5b4d6_0
  - tktable=2.10=h14c3975_0
prefix: /opt/conda/envs/R_env

R_env.yml

name: R_env
channels:
  - conda-forge
  - defaults
dependencies:
  - arrow-cpp=0.12.0
  - r-essentials=3.5.1
  - r-r.utils=2.7.0
  - r-git2r=0.24.0
prefix: /opt/conda/envs/R_env