out of core example with sqlite

sqlite.r

library(DBI)
library(RSQLite)

# create fake data
set.seed(1234)
n = 100
big_tbl = data.frame(
  ind = 1:n,
  x = runif(n),
  y = rnorm(n)
)

# write table to disk
db = dbConnect(RSQLite::SQLite(), "/tmp/db.sqlite")
dbWriteTable(db, "big_tbl", big_tbl)

# process chunks - in principle, can be done in parallel
rows_per_chunk = 7
chunks = ceiling(n / rows_per_chunk)
for (chunk in 1:chunks){
  ind_low = (chunk-1) * rows_per_chunk + 1
  ind_high = ind_low + rows_per_chunk
  query = paste("SELECT * FROM big_tbl WHERE ind >=", ind_low, "AND ind <", ind_high)
  sub_tbl = dbGetQuery(db, query)
  
  print(sub_tbl)
}

# close connection
dbDisconnect(db)