Tokenizer_Comparison.cpp

#include <string>
#include <fstream>
#include <istream>
#include <sstream>
#include <boost/tokenizer.hpp>
#include <boost/timer/timer.hpp>

using namespace std;

// 4 statements
void DoBoost( std::ofstream& cout, std::string& text )
{
    boost::char_separator<char> sep(" \n\t\r\f");
    boost::tokenizer<boost::char_separator<char>> tokens(text, sep);
    for (const auto& t : tokens) {
        cout << t ;
    }
}

// 6 statements
void DoIterator(std::ofstream& cout, std::string& str )
{
    // construct a stream from the string
    std::stringstream strstr(str);

    // use stream iterators to copy the stream to the vector as whitespace separated strings
    std::istream_iterator<std::string> it(strstr);
    std::istream_iterator<std::string> end;
    std::vector<std::string> results(it, end);

    // send the vector to stdout.
    std::ostream_iterator<std::string> oit(cout);
    std::copy(results.begin(), results.end(), oit);
}

// 4 statements
void DoIteratorCorrectly(std::ofstream& cout, std::string& str )
{
    // construct a stream from the string
    std::stringstream strstr(str);

    // use stream iterators to read individual strings
    std::istream_iterator<std::string> it(strstr);
    std::istream_iterator<std::string> end;
    
    std::for_each( it, end, [&cout]( const std::string& str ) { cout << str; } );

}

// 7 statements
void DoStrtok(std::ofstream& cout, std::string& str)
{
    char* pMutableString = (char*) malloc( str.size()+1 );
    strcpy( pMutableString, str.c_str() );
    
    char *p = strtok(pMutableString, " \n\t\r\f");
    while (p) {
        cout << p;
        p = strtok(NULL, " \n\t\r\f");
    }
    free(pMutableString);
}



// 7 statements
static bool IsDelim( char tst )
{
    const char* DELIMS = " \n\t\r\f";
    do // Delimiter string cannot be empty, so don't check for it
    {
        if( tst == *DELIMS )
            return true;
        ++DELIMS;
    } while( *DELIMS );

    return false;
}

static bool IsDelim_STDFind( char tst )
{
    // For those of you who insist that nobody can ever outperform
    //  the standard algorithms... std::find clocks in about 25% slower
    const char* DELIMS = " \n\t\r\f";
    const char* END = DELIMS+5;
    return std::find( DELIMS, END, tst ) != END;
}

// 14 statements
void DoJoshsWay( std::ofstream& cout, std::string& str)
{
    char* pMutableString = (char*) malloc( str.size()+1 );
    strcpy( pMutableString, str.c_str() );
    
    char* p = pMutableString;

    // skip leading delimiters
    while( *p && IsDelim(*p) )
        ++p;
        
    while( *p )
    {
        // note start of token
        char* pTok = p;
        
        do// skip non-delimiters
        {
            ++p;
        } while( !IsDelim(*p) && *p );

        // clobber trailing delimiter with null
        *p = 0;
        cout << pTok; // send the token

        do // skip null, and any subsequent trailing delimiters
        {
            ++p;
        } while( *p && IsDelim(*p) );
    }

    free(pMutableString);
}



int main()
{
    std::ifstream t("crytek_sponza.obj");
    std::stringstream str;
    str << t.rdbuf();
    std::string text = str.str();

    FILE* fp = fopen("times.csv", "w");
    fprintf(fp,"boost,iterators,iterators_right,strtok,josh\n");

    for( size_t i=0; i<5; i++ )
    {
        boost::timer::nanosecond_type tm;
        {
        boost::timer::auto_cpu_timer t;
        std::ofstream out_boost("out_boost.txt");
        DoBoost( out_boost, text );
        tm = t.elapsed().wall;
        } 
        fprintf(fp,"%u,", tm );
        
        {
        boost::timer::auto_cpu_timer t;
        std::ofstream out_iter("out_iter.txt");
        DoIterator( out_iter, text );
        tm = t.elapsed().wall;
        } 
        fprintf(fp,"%u,", tm );
        
        {
        boost::timer::auto_cpu_timer t;
        std::ofstream out_split("out_iter_right.txt");
        DoIteratorCorrectly( out_split, text );
        tm = t.elapsed().wall;
        } 
        fprintf(fp,"%u,", tm );
        
 
        {
        boost::timer::auto_cpu_timer t;
        std::ofstream out_strtok("out_strtok.txt");
        DoStrtok( out_strtok, text );
        tm = t.elapsed().wall;
        } 
        fprintf(fp,"%u,", tm );
        
        {
        boost::timer::auto_cpu_timer t;
        std::ofstream out_josh("out_josh.txt");
        DoJoshsWay( out_josh, text );
        tm = t.elapsed().wall;
        } 
        fprintf(fp,"%u,", tm );
        
     fprintf(fp,"\n");
    }

    return 0;
}