Memcached Benchmark which Caches a Directory and Reads it Back

memc.cpp

/*
 * sudo apt-get install memcached libmemcached-dev libboost-all-dev
 * have a look in /etc/memcached.conf and set some reasonable defaults
 * changed the following:
 * -m 4096m -I 10m
 *
 * sudo service memcached restart
 *
 * g++ memc.cpp -o memc -std=c++11 -g -O2 -lboost_system -lboost_filesystem -lmemcached
 * memc /some/directory/with/files 2147483648
 */


#include <boost/filesystem.hpp>
#include <libmemcached/memcached.hpp>

#include <chrono>
#include <fstream>
#include <string>
#include <list>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#include <iostream>

namespace {

  //get the files at the root_dir recursively
  std::list<std::string> get_files(const char* root_dir) {
    std::list<std::string> files;
    for (boost::filesystem::recursive_directory_iterator i(root_dir), end; i != end; ++i)
      if (!is_directory(i->path()))
        files.push_back(i->path().string());
    return files;
  }

  void flush_cache() {
    const char* config = "--SERVER=127.0.0.1:11211";
    auto* cache = memcached(config, strlen(config));
    memcached_flush(cache, 0);
  }

  size_t load_cache(std::list<std::string>& files, const size_t limit) {
    //connect to memcached
    const char* config = "--SERVER=127.0.0.1:11211";
    auto* cache = memcached(config, strlen(config));
    auto set = memcached_behavior_set(cache, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
    if(set != MEMCACHED_SUCCESS)
      throw std::runtime_error("Couldn't use memcached asynchronous IO mode");

    //for each file
    size_t bytes = 0;
    size_t skipped = 0;
    std::list<std::string>::iterator file_name_itr = files.begin();
    while(file_name_itr != files.end()) {

      //check the length
      std::string file_name = *file_name_itr;
      std::fstream file(file_name, std::ios_base::binary | std::ios_base::in | std::ios_base::ate);
      if(!file)
        throw std::runtime_error(file_name + "(fstream): " + strerror(errno));
      auto length = file.tellg();
      bytes += length;

      //mmap the file
      auto fd = open(file_name.c_str(), O_RDWR, 0);
      if(fd == -1)
        throw std::runtime_error(file_name + "(open): " + strerror(errno));
      void* mmap_handle = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0);
      if(mmap_handle == MAP_FAILED)
        throw std::runtime_error(file_name + "(mmap): " + strerror(errno));
      auto cl = close(fd);
      if(cl == -1)
        throw std::runtime_error(file_name + "(close): " + strerror(errno));

      //push the bytes into memcached
      auto stored = memcached_set(cache, file_name.c_str(), file_name.size(), static_cast<const char*>(mmap_handle), length, (time_t)0, (uint32_t)0);
      munmap(mmap_handle, length);
      ++file_name_itr;
      if(stored != MEMCACHED_SUCCESS) {
       files.erase(std::prev(file_name_itr));
       ++skipped;
      }
      else if(bytes > limit) {
        memcached_free(cache);
        files.erase(file_name_itr, files.end());
        return skipped;
      }
    }

    //done with it
    memcached_free(cache);
    return skipped;
  }
  float read_cache(const std::list<std::string>& files) {
    //connect to memcached
    const char* config = "--SERVER=127.0.0.1:11211";
    auto* cache = memcached(config, strlen(config));
    auto set = memcached_behavior_set(cache, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
    if(set != MEMCACHED_SUCCESS)
      throw std::runtime_error("Couldn't use memcached asynchronous IO mode");

    //for each file
    size_t miss = 0;
    for(const auto& file_name : files) {
      //pull the bytes out of memcached
      size_t length;
      uint32_t flags;
      memcached_return_t fetched;
      char* obj = memcached_get(cache, file_name.c_str(), file_name.size(), &length, &flags, &fetched);
      if(fetched != MEMCACHED_SUCCESS)
        ++miss;
      free(obj);
    }

    //done with it
    memcached_free(cache);
    return static_cast<float>(miss) / static_cast<float>(files.size());
  }
}

int main(int argc, char** argv) {

  //get the list of files in the dir
  auto files = get_files(argv[1]);

  size_t byte_limit = 1073741824;
  if(argc > 2)
    byte_limit = std::stoul(argv[2]);

  std::cout << "===Testing with " << argv[1] << "===" << std::endl;
  flush_cache();

  //load them into cache
  std::cout << "===Loading cache===" << std::endl;
  auto start = std::chrono::system_clock::now();
  auto skipped = load_cache(files, byte_limit);
  auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - start).count();
  std::cout << files.size() << " took " << elapsed << " ms" << std::endl;
  std::cout << static_cast<float>(elapsed)/static_cast<float>(files.size()) << " ms/tile" << std::endl;
  std::cout << "Skipped " << skipped << " that were too large" << std::endl;

  //read them back out
  std::cout << "===Reading cache===" << std::endl;
  start = std::chrono::system_clock::now();
  auto miss_rate = read_cache(files);
  elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - start).count();
  std::cout << files.size() << " took " << elapsed << " ms" << std::endl;
  std::cout << static_cast<float>(elapsed)/static_cast<float>(files.size()) << " ms/tile" << std::endl;
  std::cout << "Miss rate was " << miss_rate * 100.f << "%" << std::endl;

  return 0;
}