Skip to content

Instantly share code, notes, and snippets.

@boneyard93501
Created April 20, 2018 02:19
Show Gist options
  • Save boneyard93501/e9bd3edfb2d92666689bf166966aa78d to your computer and use it in GitHub Desktop.
Save boneyard93501/e9bd3edfb2d92666689bf166966aa78d to your computer and use it in GitHub Desktop.
rust rusoto S3 obj fetcher with StaticProvider
extern crate rusoto_core;
extern crate rusoto_credential;
extern crate rusoto_s3;
use rusoto_core::Region;
use rusoto_credential::StaticProvider;
use rusoto_core::reactor::RequestDispatcher;
use rusoto_s3::{S3, S3Client, ListObjectsV2Request};
fn main() {
// in lieu of toml cfg
let aws_public_key: String = "xxx".to_string();
let aws_private_key: String = "yyy".to_string();
let bucket_name: String = "abc-def-data".to_string();
let meta_prefix:String = "zzz/metadata".to_string();
let meta_postfix:String = "-onix.xml".to_string();
let region = Region::UsEast1;
let provider: StaticProvider = StaticProvider::new_minimal(aws_public_key, aws_private_key);
let s3_client = S3Client::new(RequestDispatcher::default(), provider, region);
let mut request = ListObjectsV2Request {bucket: bucket_name.clone(),
prefix: Some(meta_prefix.clone()),
..Default::default()};
let mut key_names:Vec<String> = Vec::new();
let mut key_counter = 0u32;
loop {
let result = s3_client.list_objects_v2(&request).sync().unwrap();
if !result.contents.is_some() {
break;
}
let objs = &result.contents.unwrap();
let last_obj = &objs[&objs.len() - 1];
key_counter += result.key_count.unwrap() as u32;
request.start_after = last_obj.key.clone();
for obj in objs {
if obj.key.is_some() {
let key = obj.clone().key.unwrap();
if key.ends_with(&meta_postfix) {
key_names.push(key);
}
}
else {
println!("got a borked S3 obj: {:?}", obj);
}
}
}
println!("loop counter: {}", key_counter);
println!("key count {}", key_names.len());
assert_eq!(key_names.len(), 6511);
}
/* cargo.toml
[package]
name = "ru-s3"
version = "0.1.0"
[profile.release]
lto = true
panic = 'abort'
[dependencies]
rusoto_core = "0.32.0"
rusoto_credential = "0.11"
rusoto_s3 = "0.32.0"
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment