tar_custom_format_arrow.R

# tar_custom_format_arrow.R

# store as parquet, load as an Arrow Table

format_arrow_table <- function() {targets::tar_format(
  read = function(path) {arrow::read_parquet(path, as_data_frame = FALSE)},
  write = function(object, path) {arrow::write_parquet(object, path, version = "2.0")},
  marshal = function(object) as.data.frame(object),
  unmarshal = function(object) arrow::Table$create(object),
  repository = "default"
)}

# store as an Arrow Dataset (parquet), and load as an Arrow Dataset

format_arrow_dataset <- function() {targets::tar_format(
  read = function(path) {arrow::open_dataset(path)},
  write = function(object, path) {arrow::write_dataset(object, path, version = "2.0")},
  marshal = function(object) dplyr::collect(object),
  unmarshal = function(object) arrow::Dataset$create(object),
  repository = "default"
)}