Code shared from the Rust Playground

playground.rs

extern crate walkdir;
use walkdir::WalkDir;

use std::env;
use std::f32;
use std::fs::File;
use std::io;
use std::io::{BufReader, Read};
use std::path::Path;
use std::time::Instant;

fn mean(slice: &[f32]) -> f32 {
    slice.iter().fold(0.0f32, |acc, x| acc + x) / slice.len() as f32
}

fn stddev(slice: &[f32]) -> f32 {
    let mean = mean(slice);
    (slice.iter().fold(0.0f32, |acc, x| acc + (x - mean).powi(2)) / slice.len() as f32).sqrt()
}

fn bench<'s, F: Fn(&'s str) -> io::Result<()>>(f: F, dir: &'s str, iter: usize) -> (f32, f32) {
    let times = (0..iter)
        .map(|_| {
            let now = Instant::now();
            f(dir).unwrap();
            let elapsed = now.elapsed();
            elapsed.as_secs() as f32 + (elapsed.subsec_micros() as f32 / 100_000.0)
        })
        .collect::<Vec<f32>>();
    (mean(&times), stddev(&times))
}

fn main() -> io::Result<()> {
    let mut args = env::args();

    let dir = args.nth(1).unwrap_or_else(|| "./".to_string());

    // Warm up run
    pre_allocate(&dir)?;

    for (name, func) in ["pre-allocate", "read_to_end", "bufread"]
        .iter()
        .zip([pre_allocate, read_to_end, bufread].iter())
    {
        let s = bench(func, &dir, 10);
        println!("{:15}: {} +/- {}", name, s.0, s.1);
    }
    Ok(())
}

fn pre_allocate<P: AsRef<Path>>(dir: P) -> io::Result<()> {
    let nul: u8 = 0;
    let mut bytes_count: i32;
    let mut buffer = Vec::new();

    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
        let path = entry.path();
        if !path.is_dir() {
            let mut file = File::open(&path)?;
            let len = file.metadata()?.len() as usize;

            let mut cap = buffer.capacity();
            while len > cap {
                buffer.reserve(len - cap);
                cap = buffer.capacity();
                unsafe {
                    buffer.set_len(cap);
                }
            }

            // call read() until we get 0 bytes read
            let mut read = 0;
            loop {
                let n = file.read(&mut buffer)?;
                if n == 0 {
                    break;
                }
                read += n;
            }
            assert_eq!(len, read);
            bytes_count = 0;
            for b in buffer.iter().take(read) {
                if b == &nul {
                    eprintln!("{} bytes={} binary file", path.display(), bytes_count);
                    break;
                }

                bytes_count += 1;
            }
            eprintln!("{} bytes={}", path.display(), bytes_count);
        }
    }
    Ok(())
}

fn read_to_end<P: AsRef<Path>>(dir: P) -> io::Result<()> {
    let nul: u8 = 0;
    let mut bytes_count: i32;
    let mut buffer = Vec::new();

    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
        let path = entry.path();
        if !path.is_dir() {
            let mut file = File::open(&path)?;
            let len = file.metadata()?.len() as usize;

            buffer.clear();
            let read = file.read_to_end(&mut buffer)?;
            assert_eq!(len, read);
            bytes_count = 0;
            for b in &buffer {
                if b == &nul {
                    eprintln!("{} bytes={} binary file", path.display(), bytes_count);
                    break;
                }

                bytes_count += 1;
            }
            eprintln!("{} bytes={}", path.display(), bytes_count);
        }
    }
    Ok(())
}

fn bufread<P: AsRef<Path>>(dir: P) -> io::Result<()> {
    let nul: u8 = 0;
    let mut bytes_count: i32;
    let mut buffer = Vec::with_capacity(8096);
    unsafe {
        buffer.set_len(8096);
    }

    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
        let path = entry.path();
        if !path.is_dir() {
            let mut reader = BufReader::new(File::open(&path)?);
            bytes_count = 0;
            'outer: loop {
                let n = reader.read(&mut buffer)?;
                if n == 0 {
                    break;
                }

                for b in buffer.iter().take(n) {
                    if b == &nul {
                        eprintln!("{} bytes={} binary file", path.display(), bytes_count);
                        break 'outer;
                    }
                    bytes_count += 1;
                }
            }
            eprintln!("{} bytes={}", path.display(), bytes_count);
        }
    }
    Ok(())
}