beneth.fr
/
rdupe
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implement working solution

master
Amaury Denoyelle 2018-08-07 11:04:19 +02:00 committed by Benoît Mauduit
parent 1cdc35562b
commit 351cceab7d
1 changed files with 40 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/main.rs
View File

@ -240,37 +240,50 @@ fn main() {
} }
// compute file hash in parallel // compute file hash in parallel
let magic = files_candidate.lock().unwrap().len() / num_cpus::get(); let num_cpus = num_cpus::get();
println!("Magic is : {}", magic); let files_candidate_len = files_candidate.lock().unwrap().len();
let chunk_size = (files_candidate_len / num_cpus) + 1;
let modulus = files_candidate_len % num_cpus;
for i in 0..num_cpus::get()+1 { println!("Calculate {} file(s)", files_candidate_len);
let fc = files_candidate.clone(); println!("Use {} chunk(s) of size {}", modulus, chunk_size);
let mut work = fc.lock().unwrap(); println!("Use {} chunk(s) of size {}", num_cpus - modulus, chunk_size - 1);
let end = if (i*magic)+(magic) > work.len() {
work.len() let mut guards = vec![];
} else { let fc = files_candidate.clone();
(i*magic)+(magic) let mut work = fc.lock().unwrap();
};
// Example from : // Example from :
// https://stackoverflow.com/questions/33818141/how-do-i-pass-disjoint-slices-from-a-vector-to-different-threads // https://stackoverflow.com%2Fquestions/33818141/how-do-i-pass-disjoint-slices-from-a-vector-to-different-threads
// Scoped threads allow the compiler to prove that no threads will outlive // Scoped threads allow the compiler to prove that no threads will outlive
// table (which would be bad). // table (which would be bad).
crossbeam::scope(|scope| { crossbeam::scope(|scope| {
let (split_a, split_b) = work.split_at_mut(chunk_size * modulus);
let chunk_a = split_a.chunks_mut(chunk_size);
let chunk_b = split_b.chunks_mut(chunk_size - 1);
for (i, slice) in chunk_a.chain(chunk_b).enumerate() {
// Spawn a thread operating on that subslice. // Spawn a thread operating on that subslice.
scope.spawn(move || let guard = scope.spawn(move || {
for w in &mut work[i*magic .. end] { for w in slice {
println!("[{}] Hashing : {}", i, w); println!("[{}] Hashing : {}", i, w);
if let Ok(mut file) = fs::File::open(&w.name) { //thread::sleep(std::time::Duration::from_secs(2));
w.hash::<Sha256, _>(&mut file); if let Ok(mut file) = fs::File::open(&w.name) {
} else { w.hash::<Sha256, _>(&mut file);
panic!("Error opening file (name = {})!", w.name); } else {
} panic!("Error opening file (name = {})!", w.name);
} }
); }
}); });
guards.push(guard);
}
});
for guard in guards {
guard.join();
} }
for i in files_candidate.lock().unwrap().iter() { for i in work.iter() {
println!("{}", i); println!("{}", i);
} }