Skip to content

Instantly share code, notes, and snippets.

@patternproject
Created May 6, 2017 23:46
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save patternproject/cc76f045464cee45e21017be6ebc0195 to your computer and use it in GitHub Desktop.
Save patternproject/cc76f045464cee45e21017be6ebc0195 to your computer and use it in GitHub Desktop.
purrr - doing something different in first iteration
# Helper Function
fn_downloadZip <- function(URL) {
myzip <- unz(URL, filename=basename(URL))
c.zip.name = "test.zip"
c.file.path = file.path(c.home.dir,c.data.dir,c.zip.name)
c.data.folder = file.path(c.home.dir,c.data.dir)
download.file(URL, destfile=c.file.path)
f.name = unzip(c.file.path, exdir=c.data.folder)
# readr from tidyverse supports read_csv()
df.1 = read_csv(f.name,col_names = TRUE)
## 2. Put data in the database - HAPPENS HERE
# for first time - create table .......................................... PROBLEM HERE
bikes_sqlite <-
copy_to(
dest = my_db, # remote data source
df = df.1,# local data frame
name = "my_table", # name for new remote table.
temporary = FALSE, # if TRUE, will create a temporary table that is local to this connection and will be automatically deleted when the connection expires
indexes = list("start_station_id", "gender")
)
# every time after first - insert into .......................................... PROBLEM HERE
# SRC:
# http://stackoverflow.com/questions/26568182/is-it-possible-to-insert-add-a-row-to-a-sqlite-db-table-using-dplyr-package
db_insert_into(con = my_db$con,
table = "my_table",
values = df.1)
} # end function
# List of files
c.files = c("https://s3.amazonaws.com/tripdata/201311-citibike-tripdata.zip",
"https://s3.amazonaws.com/tripdata/201312-citibike-tripdata.zip")
# Actual run
c.files %>%
map(fn_downloadZip)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment