Created
April 20, 2018 02:19
-
-
Save boneyard93501/e9bd3edfb2d92666689bf166966aa78d to your computer and use it in GitHub Desktop.
rust rusoto S3 obj fetcher with StaticProvider
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
extern crate rusoto_core; | |
extern crate rusoto_credential; | |
extern crate rusoto_s3; | |
use rusoto_core::Region; | |
use rusoto_credential::StaticProvider; | |
use rusoto_core::reactor::RequestDispatcher; | |
use rusoto_s3::{S3, S3Client, ListObjectsV2Request}; | |
fn main() { | |
// in lieu of toml cfg | |
let aws_public_key: String = "xxx".to_string(); | |
let aws_private_key: String = "yyy".to_string(); | |
let bucket_name: String = "abc-def-data".to_string(); | |
let meta_prefix:String = "zzz/metadata".to_string(); | |
let meta_postfix:String = "-onix.xml".to_string(); | |
let region = Region::UsEast1; | |
let provider: StaticProvider = StaticProvider::new_minimal(aws_public_key, aws_private_key); | |
let s3_client = S3Client::new(RequestDispatcher::default(), provider, region); | |
let mut request = ListObjectsV2Request {bucket: bucket_name.clone(), | |
prefix: Some(meta_prefix.clone()), | |
..Default::default()}; | |
let mut key_names:Vec<String> = Vec::new(); | |
let mut key_counter = 0u32; | |
loop { | |
let result = s3_client.list_objects_v2(&request).sync().unwrap(); | |
if !result.contents.is_some() { | |
break; | |
} | |
let objs = &result.contents.unwrap(); | |
let last_obj = &objs[&objs.len() - 1]; | |
key_counter += result.key_count.unwrap() as u32; | |
request.start_after = last_obj.key.clone(); | |
for obj in objs { | |
if obj.key.is_some() { | |
let key = obj.clone().key.unwrap(); | |
if key.ends_with(&meta_postfix) { | |
key_names.push(key); | |
} | |
} | |
else { | |
println!("got a borked S3 obj: {:?}", obj); | |
} | |
} | |
} | |
println!("loop counter: {}", key_counter); | |
println!("key count {}", key_names.len()); | |
assert_eq!(key_names.len(), 6511); | |
} | |
/* cargo.toml | |
[package] | |
name = "ru-s3" | |
version = "0.1.0" | |
[profile.release] | |
lto = true | |
panic = 'abort' | |
[dependencies] | |
rusoto_core = "0.32.0" | |
rusoto_credential = "0.11" | |
rusoto_s3 = "0.32.0" | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment