Created
July 10, 2021 10:31
-
-
Save 0xPranavDoshi/059faa3c0c605f03369d0f84b9c8cfb9 to your computer and use it in GitHub Desktop.
Custom MulitThreaded Download in Rust
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
async fn threaded_download(threads: u64, url: &String, output: &str) { | |
let mut handles = vec![]; | |
// Create response from url | |
let res = reqwest::get(url.to_string()).await.unwrap(); | |
// Get the total bytes of the response | |
let total_length = res.content_length().unwrap(); | |
// Create buffer for bytes | |
let mut buffer: Vec<u8> = vec![]; | |
for index in 0..threads { | |
let mut buf: Vec<u8> = vec![]; | |
let url = url.clone(); | |
let (start, end) = get_splits(index + 1, total_length, threads); | |
handles.push(tokio::spawn(async move { | |
let client = reqwest::Client::new(); | |
let mut response = client | |
.get(url) | |
.header("range", format!("bytes={}-{}", start, end)) | |
.send() | |
.await | |
.unwrap(); | |
while let Some(chunk) = response.chunk().await.unwrap() { | |
let _ = std::io::copy(&mut &*chunk, &mut buf); | |
} | |
buf | |
})) | |
} | |
// Join all handles | |
let result = futures::future::join_all(handles).await; | |
for res in result { | |
buffer.append(&mut res.unwrap().clone()); | |
} | |
let mut file = File::create(output.clone()).unwrap(); | |
let _ = file.write_all(&buffer); | |
} | |
fn get_splits(i: u64, total_length: u64, threads: u64) -> (u64, u64) { | |
let mut start = ((total_length / threads) * (i - 1)) + 1; | |
let mut end = (total_length / threads) * i; | |
if i == 1 { | |
start = 0; | |
} | |
if i == threads { | |
end = total_length; | |
} | |
(start, end) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment