bufchr vs memchr.md

bufchr vs memchr.md

sample csv file

• https://github.com/yiunsr/files/blob/main/WPP2019_TotalPopulationBySex.csv

======== Start ========
======== Test 11111 ========
test_loop        time:0.002088   count: 2534305
test_bufchr      time:0.008465   count: 2534305
test_memchr      time:0.013212   count: 2534305
======== Test 22222 ========
test_loop2       time:0.018692   count: 2815239
test_bufchr2     time:0.008988   count: 2815239
test_memchr2     time:0.016373   count: 2815239
======== Test 33333 ========
test_loop3       time:0.017076   count: 2826753
test_bufchr3     time:0.009199   count: 2826753
test_memchr3     time:0.018512   count: 2826753
======== End ========

• bufchr vs memchr : bufchr is definitely faster.

Cargo.toml

[package]
name = "search_char_ben"
version = "0.1.0"
edition = "2018"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
bufchr = { git = "https://github.com/yiunsr/bufchr", tag = "v0.3" }
# bufchr = { path =  "D:\\workspace\\vscode2\\bufchr"}
memchr = { version = "2", default-features = false }

[release]
opt-level = 3

main.rs

use std::time::Instant;
use bufchr;
use memchr;

//static CSV_HAYSTACK: &'static [u8] = include_bytes!("./data/gdp_org.csv");
static CSV_HAYSTACK: &'static [u8] = include_bytes!("./data/WPP2019_TotalPopulationBySex.csv");
static TEST_COUNT:i32 = 100;

fn boot_haystack(){
    let mut count_ = 0;
    for _ in CSV_HAYSTACK{
        count_ += 1;
    }
}

fn test_loop(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    for _ in 0..TEST_COUNT {
        for ch in CSV_HAYSTACK{
            if *ch == n1{
                count += 1;
            }
        }
    }
    println!("test_loop\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}


fn test_bufchr(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut bf = bufchr::Bufchr::new(CSV_HAYSTACK, n1);
        loop {
            let n = bf.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_bufchr\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn test_memchr(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut it = memchr::memchr_iter(n1, CSV_HAYSTACK);
        loop {
            let n = it.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_memchr\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn test_loop2(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    for _ in 0..TEST_COUNT {
        for ch in CSV_HAYSTACK{
            if (*ch == n1) | (*ch == n2){
                count += 1;
            }
        }
    }
    println!("test_loop2\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}


fn test_bufchr2(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut bf = bufchr::Bufchr2::new(CSV_HAYSTACK, n1, n2);
        loop {
            let n = bf.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_bufchr2\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn test_memchr2(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut it = memchr::memchr2_iter(n1, n2,CSV_HAYSTACK);
        loop {
            let n = it.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_memchr2\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn test_loop3(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        for ch in CSV_HAYSTACK{
            if (*ch == n1) | (*ch == n2) | (*ch == n3){
                count += 1;
            }
        }
    }
    println!("test_loop3\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}


fn test_bufchr3(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut bf = bufchr::Bufchr3::new(CSV_HAYSTACK, n1, n2, n3);
        loop {
            let n = bf.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_bufchr3\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn test_memchr3(){
    let before = Instant::now();
    let mut count = 0;
    let n1 = b',';
    let n2 = b'\n';
    let n3 = b'"';
    for _ in 0..TEST_COUNT {
        let mut it = memchr::memchr3_iter(n1, n2, n3, CSV_HAYSTACK);
        loop {
            let n = it.next();
            if n == None{break;}
            count +=1;
        }
    }
    println!("test_memchr3\t time:{:.6?} \t count: {}", 
        before.elapsed().as_secs_f64() / TEST_COUNT as f64, count/TEST_COUNT);
}

fn main() {
    println!("======== Start ========");
    boot_haystack();
    println!("======== Test 11111 ========");
    test_loop();
    test_bufchr();
    test_memchr();
    println!("======== Test 22222 ========");
    test_loop2();
    test_bufchr2();
    test_memchr2();
    println!("======== Test 33333 ========");
    test_loop3();
    test_bufchr3();
    test_memchr3();
    println!("======== End ========");
}