Skip to content

Instantly share code, notes, and snippets.

@mooreniemi
Created May 7, 2024 14:18
Show Gist options
  • Save mooreniemi/ebe1ea619abed76c22dd3228fcde2b21 to your computer and use it in GitHub Desktop.
Save mooreniemi/ebe1ea619abed76c22dd3228fcde2b21 to your computer and use it in GitHub Desktop.
This is a minimal example of reading a delta table from S3 with Rust. From [this discussion](https://delta-users.slack.com/archives/C013LCAEB98/p1714949120391979).
[package]
name = "read-table-example"
version = "0.1.0"
edition = "2021"
# note: I did not double-check this and your dependencies might slightly vary (I cut some of mine off)
[dependencies]
deltalake = { version = "*", features = ["s3"] }
tokio = { version = "1", features = ["full"] }
quick-xml = { version = "0.23.0-alpha3" }
aws-config = { version = "*", features = ["rustls"] }
aws-credential-types = "*"
aws-types = "*"
use deltalake::open_table_with_storage_options;
use std::{collections::HashMap, error::Error};
use aws_credential_types::provider::ProvideCredentials;
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
let table_uri = "s3://my-valid-table";
// I am using a specific profile
let sdk_config = aws_config::load_from_env().await;
let cp = sdk_config
.credentials_provider().expect("credentials");
let creds = cp.provide_credentials().await?;
let mut options = HashMap::new();
options.insert("AWS_ACCESS_KEY_ID".into(), creds.access_key_id().to_string());
options.insert("AWS_SECRET_ACCESS_KEY".into(), creds.secret_access_key().to_string());
options.insert("AWS_SESSION_TOKEN".into(), creds.session_token().expect("got token").to_string());
// you need to be psychic to know to do this https://github.com/delta-io/delta-rs/releases/tag/rust-v0.17.0
deltalake::aws::register_handlers(None);
let dt = open_table_with_storage_options(table_uri, options).await.expect("got table");
println!("{:?} version: {}", table_uri, dt.version());
Ok(())
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment