mmap vs posix write perf benchmarking

Makefile

all:test

CPPFLAGS+=-std=c++1y -Wall -pedantic
CPPFLAGS+=-g -O2
#BOOST_DIR=/mnt/LARGE/MODULAR_BOOST/modular-boost
BOOST_DIR=/home/sehe/custom/boost
CPPFLAGS+=-isystem /home/sehe/custom/nonius/include
CPPFLAGS+=-isystem $(BOOST_DIR)

# CPPFLAGS+=-fopenmp
CPPFLAGS+=-pthread
CPPFLAGS+=-march=native

LDFLAGS+=-L $(BOOST_DIR)/stage/lib/ -Wl,-rpath,$(BOOST_DIR)/stage/lib
LDFLAGS+=-lboost_system -lboost_regex -lboost_thread -lboost_iostreams -lboost_serialization
LDFLAGS+=-lboost_date_time -lboost_chrono

CXX=g++-5
# CXX=/usr/lib/gcc-snapshot/bin/g++
# CC=/usr/lib/gcc-snapshot/bin/gcc
# CXX=clang++-3.6 -stdlib=libc++
# CC=clang

%.S:%.cpp
	$(CXX) $(CPPFLAGS) $^ -S -masm=intel -o - | egrep -v '\s*\.' | c++filt > $@

%.o:%.cpp
	$(CXX) $(CPPFLAGS) $^ -c -o $@

%:%.o
	$(CXX) $(CPPFLAGS) $^ -o $@ $(LDFLAGS)

test.cpp

#include <boost/chrono.hpp>
#include <boost/chrono/chrono_io.hpp>
#include <iostream>
#include <vector>
#include <algorithm>

// mmap the manual way
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#ifndef COLIRU
const std::size_t TOTAL_SIZE = 5ul << 30;
const std::size_t BLOCK_SIZE = 1ul << 20;
#else
const std::size_t TOTAL_SIZE = 1ul << 20;
const std::size_t BLOCK_SIZE = 1ul <<  9;
#endif
static_assert(0 == TOTAL_SIZE%BLOCK_SIZE, "not divisable by block size");
const int N = TOTAL_SIZE/BLOCK_SIZE;

template <typename Caption, typename F>
auto timed(Caption const& task, F&& f) {
    using namespace boost::chrono;
    struct _ {
        high_resolution_clock::time_point s;
        Caption const& task;
        ~_() { std::cout << " -- (" << task << " completed in " << duration_cast<milliseconds>(high_resolution_clock::now() - s) << ")\n"; }
    } timing { high_resolution_clock::now(), task };

    return f();
}

void test_mapped_file() {
    std::vector<char> const data(BLOCK_SIZE, 1);
    const std::string filename = "test_filemapping.dat";

    std::remove(filename.c_str());

    int fd = open(filename.c_str(), O_RDWR|O_CREAT, 0644);

    if (fd==-1) {
        perror("open");
        exit(255);
    }

    if(posix_fallocate64(fd, 0, TOTAL_SIZE)) {
        perror("fallocate64");
        exit(255);
    }

    posix_fadvise64(fd, 0, TOTAL_SIZE, POSIX_FADV_WILLNEED | POSIX_FADV_SEQUENTIAL);

    char* fmap = static_cast<char*>(mmap64(nullptr, TOTAL_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0));

    if (!fmap || fmap == MAP_FAILED) {
        perror("mmap");
        exit(255);
    }

    madvise(fmap, TOTAL_SIZE, MADV_SEQUENTIAL | MADV_WILLNEED);

    timed(filename, [output=fmap, &data] () mutable {
        for (size_t i = 0; i < N; i++) {
            std::copy_n(data.data(), data.size(), output);
            output += data.size();
        }
    });

    munmap(fmap, TOTAL_SIZE);
    close(fd);
}

void test_posix_write() {
    std::vector<char> const data(BLOCK_SIZE, 1);
    const std::string filename = "test_posix.dat";

    std::remove(filename.c_str());

    int fd = open(filename.c_str(), O_RDWR|O_CREAT, 0644);

    if (fd==-1) {
        perror("open");
        exit(255);
    }

    posix_fadvise64(fd, 0, TOTAL_SIZE, POSIX_FADV_WILLNEED | POSIX_FADV_SEQUENTIAL);

    timed(filename, [&] () mutable {
        for (size_t i = 0; i < N; i++) {
            ptrdiff_t count = ::write(fd, data.data(), data.size());
            if (-1 == count) { 
                perror("write");
                exit(255);
            }
            assert(count == BLOCK_SIZE);
        }
    });

    close(fd);
}

int main() {
    test_mapped_file();
    test_posix_write();
}