arrow you are ridiculous right now...

duckdb-forecasts.R

library(duckdbfs)
library(dplyr)
remotes::install_github("cboettig/duckdbfs")

library(arrow)
s3 <- s3_bucket("tern4cast-inventory", endpoint_override = "data.ecoforecast.org")
s3$ls("tern4cast-forecasts")

df <- open_dataset(s3$path("tern4cast-forecasts")) |> collect()
names(df) <- c("parquet", "theme", "model_id", "reference_datetime", "date", "part") 

urls <- df |> filter(part == "part-0.parquet") |> 
  mutate(urls = paste("https://data.ecoforecast.org/tern4cast-forecasts",
                   parquet, theme, model_id, reference_datetime, date, part, sep="/")) |>
  pull(urls)


ds <- open_dataset(urls)