Last active
April 29, 2019 05:38
-
-
Save Ironlenny/3f62abbef613bae2d8bb130c7fbf18f4 to your computer and use it in GitHub Desktop.
Creating `Arc`s Inside `rayon` Threads
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// First Stage: Create file ids and partial bodies for FileDescription. Send | |
// file ids, partial bodies and file readers to the correct channels. | |
fn create_file_id(&self, files: Vec<PathBuf>) -> Result<(), ExitFailure> { | |
let (tx_main, _) = &self.main; // sender for create_main() | |
let (tx_input, _) = &self.input; // sender for create_input() | |
let (tx_fd, _) = &self.file_description; // sender for create_fd() | |
for file in files { | |
let tx_main = tx_main.clone(); | |
let tx_input = tx_input.clone(); | |
let tx_fd = tx_fd.clone(); | |
// Spawn thread | |
spawn(move || { | |
let mut reader = File::open(&file) | |
.with_context(|_| format!("Could not open file {}", file.display())) | |
.unwrap(); | |
// Get filename from path | |
let name = file | |
.file_stem() | |
.unwrap() | |
.to_string_lossy() | |
.into_owned() | |
.into_bytes(); | |
let length = { | |
let metadata = metadata(&file).unwrap(); | |
metadata.len() | |
}; | |
// Hash first 16k of the file | |
let hash_16k = { | |
let mut hasher_16k = Md5::new(); | |
let mut buffer = [0; 16384]; | |
reader.read(&mut buffer).unwrap(); | |
for byte in buffer.iter() { | |
hasher_16k.input([byte.clone()]); | |
} | |
let result = hasher_16k.result(); | |
let hash_16k = result.as_slice().to_owned(); | |
hash_16k | |
}; | |
// Generate File ID | |
let file_id = { | |
let mut hasher_file_id = Md5::new(); | |
hasher_file_id.input(&hash_16k); | |
hasher_file_id.input(&length.to_le_bytes()); | |
hasher_file_id.input(&name); | |
let file_id = hasher_file_id.result().to_vec(); | |
let file_id = self.convert_to_byte_array(file_id); | |
Arc::new(file_id) | |
}; | |
// Partial FileDescription (name, length, hash_16k) | |
let partial_body = (name, length, hash_16k); | |
// sender for channels | |
tx_main.send(Arc::clone(&file_id)).unwrap(); | |
tx_input.send((Arc::clone(&file_id), file, length)).unwrap(); | |
tx_fd.send(partial_body).unwrap(); | |
}); | |
} | |
Ok(()) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment