trouble.rs

extern crate hyper;
extern crate flate2;

use hyper::Client;
use std::io::{BufRead, BufReader};
use flate2::read::GzDecoder;

fn main() {
    let client = Client::new();
    let res = client.get("https://aws-publicdatasets.s3.amazonaws.com/common-crawl/crawl-data/CC-MAIN-2015-40/segments/1443736672328.14/wat/CC-MAIN-20151001215752-00000-ip-10-137-6-227.ec2.internal.warc.wat.gz")
                        .send().unwrap();

    let decoder = GzDecoder::new(res).unwrap();
    let reader = BufReader::new(decoder);
    for c in reader.lines() {
        println!("{}", c.unwrap());
    }
}
/*
only outputs a few lines, should spew megabytes:
WARC/1.0
WARC-Type: warcinfo
WARC-Date: 2015-11-08T20:11:50Z
WARC-Filename: CC-MAIN-20151001215752-00000-ip-10-137-6-227.ec2.internal.warc.gz
WARC-Record-ID: <urn:uuid:64b2a8b0-ad11-406f-8619-0d615ff46a0b>
Content-Type: application/warc-fields
Content-Length: 108

Software-Info: ia-web-commons.1.0-SNAPSHOT-20151107044109
Extracted-Date: Sun, 08 Nov 2015 20:11:50 GMT

*/