Skip to content

Instantly share code, notes, and snippets.

@sbeckeriv
Created June 3, 2016 06:00
Show Gist options
  • Save sbeckeriv/66269d386ea387d051504972fa9f0d3c to your computer and use it in GitHub Desktop.
Save sbeckeriv/66269d386ea387d051504972fa9f0d3c to your computer and use it in GitHub Desktop.
rust scraping
extern crate curl;
use curl::easy::Easy;
use std::error::Error;
use std::io::prelude::*;
use std::fs::File;
use std::path::Path;
// Print a web page onto stdout
fn main() {
for i in 500..99951{
let p =format!("zips/{:0>5}.txt",i);
println!("{}",p);
let path = Path::new(&p);
let display = path.display();
// Open a file in write-only mode, returns `io::Result<File>`
let mut file = File::create(&path).unwrap();
let url = format!("http://blah?ZIP={:0>5}",i);
println!("{}",url);
let mut buf = Vec::new();
let mut handle = Easy::new();
handle.url(&url).unwrap();
let mut transfer = handle.transfer();
transfer.write_function(|data| {
buf.extend_from_slice(data);
file.write_all(&buf).unwrap();
Ok(data.len())
}).unwrap();
transfer.perform().unwrap();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment