Skip to content

Instantly share code, notes, and snippets.

@0xPranavDoshi
Created July 10, 2021 10:31
Show Gist options
  • Save 0xPranavDoshi/059faa3c0c605f03369d0f84b9c8cfb9 to your computer and use it in GitHub Desktop.
Save 0xPranavDoshi/059faa3c0c605f03369d0f84b9c8cfb9 to your computer and use it in GitHub Desktop.
Custom MulitThreaded Download in Rust
async fn threaded_download(threads: u64, url: &String, output: &str) {
let mut handles = vec![];
// Create response from url
let res = reqwest::get(url.to_string()).await.unwrap();
// Get the total bytes of the response
let total_length = res.content_length().unwrap();
// Create buffer for bytes
let mut buffer: Vec<u8> = vec![];
for index in 0..threads {
let mut buf: Vec<u8> = vec![];
let url = url.clone();
let (start, end) = get_splits(index + 1, total_length, threads);
handles.push(tokio::spawn(async move {
let client = reqwest::Client::new();
let mut response = client
.get(url)
.header("range", format!("bytes={}-{}", start, end))
.send()
.await
.unwrap();
while let Some(chunk) = response.chunk().await.unwrap() {
let _ = std::io::copy(&mut &*chunk, &mut buf);
}
buf
}))
}
// Join all handles
let result = futures::future::join_all(handles).await;
for res in result {
buffer.append(&mut res.unwrap().clone());
}
let mut file = File::create(output.clone()).unwrap();
let _ = file.write_all(&buffer);
}
fn get_splits(i: u64, total_length: u64, threads: u64) -> (u64, u64) {
let mut start = ((total_length / threads) * (i - 1)) + 1;
let mut end = (total_length / threads) * i;
if i == 1 {
start = 0;
}
if i == threads {
end = total_length;
}
(start, end)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment