scoring-speed-benchmarks.R

scoring-speed.R

library(score4cast)
library(arrow)
readRenviron(path.expand("~/.Renviron"))
Sys.setenv("AWS_EC2_METADATA_DISABLED"="TRUE")
Sys.unsetenv("AWS_DEFAULT_REGION")
endpoint = "data.ecoforecast.org"
s3_forecasts <- arrow::s3_bucket("neon4cast-forecasts", endpoint_override = endpoint)
s3_targets <- arrow::s3_bucket("neon4cast-targets", endpoint_override = endpoint)
s3_scores_path <- tempfile() # dummy location for testing


theme <- "terrestrial_30min"
fc <- arrow::open_dataset(s3_forecasts$path(glue::glue("parquet/{theme}"))) 

## these steps are kinda slow, but done only once for the whole theme and don't get much slower as we get more and more forecasts
grouping <- fc |> 
  dplyr::distinct(model_id, reference_datetime, site_id) |>
  dplyr::collect()
n <- nrow(grouping)

target <- score4cast:::get_target(theme, s3_targets)



## example for each i
i <- n - 5 # consider just one


group <- grouping[i,]
ref <- lubridate::as_date(group$reference_datetime)
bounds <- list(min = ref, max = ref + max_horizon)
tg <- target |>
  filter(datetime >= bounds$min,
         datetime <= bounds$max)


bench::bench_time({
  fc_i <- fc |> 
    dplyr::filter(model_id == group$model_id, 
                  reference_datetime == group$reference_datetime,
                  site_id == group$site_id) |> 
    dplyr::collect() # |>
    ## let's try using only the most recent day
    # dplyr::filter(datetime < lubridate::as_datetime("2022-11-21")) 
})

bench::bench_time({ 
  fc_i |> 
    filter(!is.na(family)) |>
    crps_logs_score(tg) |>
    arrow::write_dataset(s3_scores_path,
                         partitioning = c("model_id",
                                          "reference_datetime",
                                          "site_id"))
  
})