Skip to content

Instantly share code, notes, and snippets.

@nicholasf
Created December 21, 2016 03:30
Show Gist options
  • Save nicholasf/c14882a8ada491a1ffc9309415371303 to your computer and use it in GitHub Desktop.
Save nicholasf/c14882a8ada491a1ffc9309415371303 to your computer and use it in GitHub Desktop.
extern crate rusoto;
extern crate chrono;
use chrono::*;
use std::default::Default;
use std::io::prelude::*;
use std::fs::File;
use std::fs::OpenOptions;
use rusoto::{ProfileProvider, Region};
use rusoto::s3::{S3Client, ListObjectsRequest, GetObjectRequest};
fn main() {
println!("Beginning.");
// let client = get_client();
let mut provider = ProfileProvider::new().unwrap();
provider.set_file_path("/Users/nicholasf/.aws/credentials.properties");
let client = S3Client::new(provider, Region::ApSoutheast2);
// let bucket = "s3://ops-prod-logging-1/tcog/prod/AWSLogs/877800914193/elasticloadbalancing/ap-southeast-2/2016/12/18".to_string();
let bucket = "ops-prod-logging-1".to_string();
let prefix = Some("tcog/prod/AWSLogs/877800914193/elasticloadbalancing/ap-southeast-2/2016/12/19".to_string());
let delimiter = Some("|".to_string());
let mut log_files = Vec::new();
let list_objects_request: ListObjectsRequest = ListObjectsRequest {
bucket: bucket,
prefix: prefix,
max_keys: Some(10),
delimiter: delimiter,
marker: None,
encoding_type: None,
};
println!("Ok, requesting the list of objects.");
let mut object_request: GetObjectRequest = Default::default();
object_request.bucket = "ops-prod-logging-1".to_string();
match client.list_objects(&list_objects_request) {
Ok(output) => {
log_files = output.contents;
}
Err(error) => {
println!("Error: {:?}", error);
}
}
for content in log_files {
println!("{}", content.key);
object_request.key = content.key;
let mut provider2 = ProfileProvider::new().unwrap();
provider2.set_file_path("/Users/nicholasf/.aws/credentials.properties");
let log_file_client = S3Client::new(provider2, Region::ApSoutheast2);
match log_file_client.get_object(&object_request) {
Ok(output) => {
let log = String::from_utf8(output.body).unwrap();
let csv = generate_csv(log);
match OpenOptions::new().write(true).append(true).open("out.csv") {
Ok(mut f) => {
for line in csv {
match f.write_all(line.as_bytes()) {
Ok(_) => {
println!("Wrote out the csv");
}
Err(err) => {
println!("Error: {:?}", err);
}
}
}
}
Err(err) => {
println!("Error: {:?}", err);
}
}
}
Err(error) => {
println!("Error: {:?}", error);
}
}
}
}
// fn get_client() -> rusoto::s3::S3Client<P,D> {
// let mut provider = ProfileProvider::new().unwrap();
// provider.set_file_path("/Users/nicholasf/.aws/credentials.properties");
// return S3Client::new(provider, Region::ApSoutheast2);
// }
//fn get_object_list(bucket &str, max_keys i32) => Vec s3:Object]{}
// let mut csv = Vec::new();
fn generate_csv(log: String) -> Vec<String> {
let mut log_lines: Vec<&str> = log.split("- -").collect();
let mut csv: Vec<String> = Vec::new();
for line in log_lines {
match parse_log_into_csv(line.to_string()) {
Ok(csv_line) => {
println!("{}", csv_line);
csv.push(csv_line);
}
Err(_) => {
println!("Malformed!");
}
}
}
return csv;
}
fn parse_log_into_csv(line: String) -> Result<String, &'static str> {
let mut v: Vec<&str> = line.split(" ").collect();
if v.len() < 14 || v[13].len() == 0 {
println!(">> {}", line);
return Err("Malformed.");
}
// let epoch_timespec = time::get_time(v[1]);
// let mut t = epoch_timespec.sec.to_string().to_owned();
// let nanosecs = epoch_timespec.nsec.to_string();
// t.push(nanosecs);
match v[0].parse::<DateTime<UTC>>() {
Ok(t) => {
let mut entry = t.timestamp().to_string();
let mut millis = t.timestamp_subsec_millis().to_string();
if millis.len() == 0 {
millis = "000".to_string();
}
entry.push_str(millis.as_str());
entry.push_str(" \t ");
let uri = v[12].replace("http://tcog.news.com.au:80", "");
entry.push_str(uri.as_str());
entry.push_str("\n");
println!("{}", entry);
return Ok(entry);
}
Err(err) => {
println!("err!");
println!("{:?}", err);
return Err("Malformed");
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment