Skip to content

Instantly share code, notes, and snippets.

@lsk569937453
Last active May 11, 2024 02:50
Show Gist options
  • Save lsk569937453/e0496754cf3ebe740a33a7759516015e to your computer and use it in GitHub Desktop.
Save lsk569937453/e0496754cf3ebe740a33a7759516015e to your computer and use it in GitHub Desktop.
multi thread write to file
[package]
name = "reqwest_download"
version = "0.1.0"
edition = "2021"
[dependencies]
tokio = { version = "~1.34.0", features = ["full"] } # Event-driven, non-blocking I/O platform.
anyhow="1.0"
reqwest = { version = "0.12", features = ["stream"] }
futures = "0.3"
openssl = { version = "0.10", features = ["vendored"] }
futures-util = "0.3.30"
use std::fmt::format;
use std::time::Duration;
use anyhow::ensure;
use futures_util::StreamExt;
use reqwest::Client;
use tokio::fs::File;
use tokio::io::SeekFrom;
use tokio::io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::task::JoinSet;
use tokio::time::{sleep, Instant};
#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
let now = Instant::now();
test1().await;
println!("{}ms", now.elapsed().as_millis());
Ok(())
}
async fn test1() -> Result<(), anyhow::Error> {
let mut set = JoinSet::new();
let byte_array: Vec<u8> = (0..=1024 * 1024).map(|x| x as u8).collect();
for start in 0..50 {
let cloned_bytes = byte_array.clone();
set.spawn(async move {
download_file1(start, cloned_bytes).await;
});
}
while let Some(res) = set.join_next().await {
let out = res?;
}
Ok(())
}
async fn test2() -> Result<(), anyhow::Error> {
let mut set = JoinSet::new();
let byte_array: Vec<u8> = (0..=1024 * 1024).map(|x| x as u8).collect();
for start in 0..50 {
let cloned_bytes = byte_array.clone();
set.spawn(async move {
download_file2(start, cloned_bytes).await;
});
}
while let Some(res) = set.join_next().await {
let out = res?;
}
Ok(())
}
async fn download_file1(start: i32, byte_array: Vec<u8>) -> Result<(), anyhow::Error> {
let mut file = tokio::fs::OpenOptions::new()
.create(true)
.write(true)
.open("test.bin")
.await?;
let current = start * byte_array.len() as i32;
sleep(Duration::from_secs(10)).await;
file.seek(SeekFrom::Start(current as u64)).await;
file.write_all(&byte_array).await;
Ok(())
}
async fn download_file2(start: i32, byte_array: Vec<u8>) -> Result<(), anyhow::Error> {
let file_name = format!("{}.bin", start);
let mut file = tokio::fs::OpenOptions::new()
.create(true)
.write(true)
.open(file_name)
.await?;
let current = start * byte_array.len() as i32;
sleep(Duration::from_secs(10)).await;
file.write_all(&byte_array).await;
Ok(())
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment