Skip to content

Instantly share code, notes, and snippets.

@Ironlenny
Last active April 29, 2019 05:38
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Ironlenny/3f62abbef613bae2d8bb130c7fbf18f4 to your computer and use it in GitHub Desktop.
Save Ironlenny/3f62abbef613bae2d8bb130c7fbf18f4 to your computer and use it in GitHub Desktop.
Creating `Arc`s Inside `rayon` Threads
// First Stage: Create file ids and partial bodies for FileDescription. Send
// file ids, partial bodies and file readers to the correct channels.
fn create_file_id(&self, files: Vec<PathBuf>) -> Result<(), ExitFailure> {
let (tx_main, _) = &self.main; // sender for create_main()
let (tx_input, _) = &self.input; // sender for create_input()
let (tx_fd, _) = &self.file_description; // sender for create_fd()
for file in files {
let tx_main = tx_main.clone();
let tx_input = tx_input.clone();
let tx_fd = tx_fd.clone();
// Spawn thread
spawn(move || {
let mut reader = File::open(&file)
.with_context(|_| format!("Could not open file {}", file.display()))
.unwrap();
// Get filename from path
let name = file
.file_stem()
.unwrap()
.to_string_lossy()
.into_owned()
.into_bytes();
let length = {
let metadata = metadata(&file).unwrap();
metadata.len()
};
// Hash first 16k of the file
let hash_16k = {
let mut hasher_16k = Md5::new();
let mut buffer = [0; 16384];
reader.read(&mut buffer).unwrap();
for byte in buffer.iter() {
hasher_16k.input([byte.clone()]);
}
let result = hasher_16k.result();
let hash_16k = result.as_slice().to_owned();
hash_16k
};
// Generate File ID
let file_id = {
let mut hasher_file_id = Md5::new();
hasher_file_id.input(&hash_16k);
hasher_file_id.input(&length.to_le_bytes());
hasher_file_id.input(&name);
let file_id = hasher_file_id.result().to_vec();
let file_id = self.convert_to_byte_array(file_id);
Arc::new(file_id)
};
// Partial FileDescription (name, length, hash_16k)
let partial_body = (name, length, hash_16k);
// sender for channels
tx_main.send(Arc::clone(&file_id)).unwrap();
tx_input.send((Arc::clone(&file_id), file, length)).unwrap();
tx_fd.send(partial_body).unwrap();
});
}
Ok(())
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment