Metadata load latency benchmark snippet for Lance, Delta Lake, and Iceberg

metadata-load-benchmark-snippet.rs

// Abridged from the Rust harness used for the Lance / Delta Lake / Iceberg benchmark.
// This is measured separately from commit latency. Each load starts from a fresh
// table/dataset handle and reads metadata up to the data file paths.

async fn lance_load(config: &BenchConfig) -> Result<Duration> {
    let load_session = session();
    let mut builder = DatasetBuilder::from_uri(&config.table_uri).with_session(load_session);
    if let Some(params) = store_params(config) {
        builder = builder.with_store_params(params);
    }

    let start = Instant::now();
    let dataset = builder
        .load()
        .await
        .with_context(|| format!("load Lance dataset {}", config.table_uri))?;
    let _fragments = dataset.manifest().fragments.len();
    Ok(start.elapsed())
}

async fn delta_load(
    config: &BenchConfig,
    storage_options: &HashMap<String, String>,
    storage_backend: Option<&std::sync::Arc<object_store::DynObjectStore>>,
) -> Result<Duration> {
    let table_url = url::Url::parse(&config.table_uri)?;
    let mut builder = deltalake::table::builder::DeltaTableBuilder::from_url(table_url.clone())?
        .with_storage_options(storage_options.clone());
    if let Some(storage_backend) = storage_backend {
        builder = builder.with_storage_backend(std::sync::Arc::clone(storage_backend), table_url);
    }

    let start = Instant::now();
    let _table = builder
        .load()
        .await
        .with_context(|| format!("load Delta table {}", config.table_uri))?;
    Ok(start.elapsed())
}

async fn iceberg_load(
    catalog: &dyn Catalog,
    ident: &TableIdent,
) -> Result<Duration> {
    let start = Instant::now();
    let table = catalog
        .load_table(ident)
        .await
        .with_context(|| format!("load Iceberg table {ident}"))?;
    // Unlike Lance and Delta Lake, use an empty scan to load the necessary metadata.
    let scan = table.scan().select_empty().build()?;
    let mut files = scan.plan_files().await?;
    while files.try_next().await?.is_some() {}
    Ok(start.elapsed())
}