s3-provider-benchmarks.R

## Using Forecasts data, ~ 300+ GB in many very small partitions!

uri <- "s3://anonymous@bio230014-bucket01/neon4cast-forecasts/parquet/aquatics?endpoint_override=sdsc.osn.xsede.org"
bench::bench_time(df <- duckdbfs::open_dataset(uri))
# process    real 
#   1.23m   4.91m

bench::bench_time( df <- arrow::open_dataset(uri))
# process    real 
#  4.21m  32.46m 

## Using scores data, ~ 10 GB

uri <- "s3://anonymous@bio230014-bucket01/neon4cast-scores/parquet/aquatics?endpoint_override=sdsc.osn.xsede.org"
bench::bench_time(df <- duckdbfs::open_dataset(uri))
# process    real 
#   2.66s  10.81s 

bench::bench_time( df <- arrow::open_dataset(uri))
# process    real 
#     12s   41.5s 

uri <- "s3://anonymous@us-west-2.opendata.source.coop/eco4cast/neon4cast-scores/parquet/aquatics?region=us-west-2"
bench::bench_time(
  df <- duckdbfs::open_dataset(uri)
)
# process    real
#   4.95s   19.6s

bench::bench_time(
  df <- arrow::open_dataset(uri)
)
# process    real
#  36.27s   1.48m

uri <- "s3://neon4cast-scores/parquet/aquatics?endpoint_override=data.ecoforecast.org"
bench::bench_time(
  df <- duckdbfs::open_dataset(uri)
)
# process    real
# 4.3s   42.7s