illumos mmap latency test program

mmap_test.c

/* MMAP testing utility
 *
 * Build: gcc -o mmap_test mmap_test.c
 *
 * Create the test files:
 *   mkdir test_data
 *   for i in {1..1000} ; do \
 *     head -c 1000000 < /dev/urandom > test_data/test_data${i}.bin
 *   done
 *
 * Usage:
 *   # ulimit -n unlimited
 *   # ./mmap_test <num_threads> ./test_data
 */


#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/types.h>
#include <fcntl.h>
#include <strings.h>
#include <sys/mman.h>
#include <errno.h>
#include <unistd.h>

#define THE_SIZE 1000000

typedef struct {
  char *directory;
  int num;
} closure_t;

uint64_t *mmap_time;
uint64_t max_time = 0;
void **buffers;
void **anon_buffers;
size_t *map_sizes;

static inline void sub_timeval(struct timeval a, struct timeval b,
                               struct timeval *out)
{
  out->tv_usec = a.tv_usec - b.tv_usec;
  if (out->tv_usec < 0L) {
    a.tv_sec--;
    out->tv_usec += 1000000L;
  }
  out->tv_sec = a.tv_sec - b.tv_sec;
  if (out->tv_sec < 0L) {
    out->tv_sec++;
    out->tv_usec -= 1000000L;
  }
}

void *run_thread (void *closure) {
  closure_t *data = (closure_t *)closure;
  int num = data->num;
  char *directory = data->directory;
  int fd;
  struct timeval stop, start, diff;
  char file[65535];
  void *buff;
  uint64_t rando = rand() % (1000000) + 500000;

  /* Sleep for some delay to stagger this thread's work relative to others */
  usleep(rando);
  sprintf(file, "%s/test_data_%d.bin", directory, num+1);

  fd = open(file, O_RDONLY);
  if (fd < 0) {
    printf("ERROR: Couldn't open file %s (%s)\n", file, strerror(errno));
    return NULL;
  }

  anon_buffers[num] = mmap(NULL, map_sizes[num], PROT_WRITE|PROT_READ, MAP_PRIVATE|MAP_ANON, -1, 0);
  if (anon_buffers[num] == MAP_FAILED) {
    printf("Anon map failed: %s\n", strerror(errno));
    close(fd);
    return NULL;
  }

  /* More jitter */
  rando = rand() % (1000000) + 500000;
  usleep(rando);
  gettimeofday(&start, NULL);
  buffers[num] = (void *)mmap(NULL, map_sizes[num], PROT_READ, MAP_PRIVATE, fd, 0);

  if (buffers[num] == MAP_FAILED) {
    printf("ERROR: Couldn't mmap fd %d, file %s (%s)\n", fd, file, strerror(errno));
    close(fd);
    return NULL;
  }

  /* We typically send whole files from start to finish, so hint at that pattern */
  posix_madvise(buffers[num], map_sizes[num], POSIX_MADV_SEQUENTIAL);
  gettimeofday(&stop, NULL);

  /* Make use of the mapped buffers to trigger actual work
   * In the real case we'd be sending this to a socket
   */
  memcpy(anon_buffers[num], buffers[num], map_sizes[num]);

  sub_timeval(stop, start, &diff);
  close(fd);

  mmap_time[num] = (diff.tv_sec * 1000000) + (diff.tv_usec);
  free(directory);
  free(data);
}

int main(int argc, char **argv)
{
  int i = 0;
  int num_threads;
  char *directory;
  pthread_t *tid;

  if (argc != 3) {
    printf("ERROR: Need 2 args - <num threads> <directory>\n");
    exit(0);
  }

  num_threads = atoi(argv[1]);
  directory = strdup(argv[2]);
  tid = (pthread_t *)calloc(num_threads, sizeof(pthread_t));
  mmap_time = (uint64_t *)calloc(num_threads, sizeof(uint64_t));
  buffers = (void *)calloc(num_threads, sizeof(void *));
  anon_buffers = (void *)calloc(num_threads, sizeof(void *));
  map_sizes = (size_t *)calloc(num_threads, sizeof(int));

  for (i=0; i<num_threads; i++) {
    map_sizes[i] = THE_SIZE;
  }

  /* Spawn some threads and mmap stuff */
  for (i=0; i<num_threads; i++) {
    closure_t *closure = (closure_t *)calloc(1, sizeof(closure_t));
    closure->num = i;
    closure->directory = strdup(directory);
    if (pthread_create(&tid[i], NULL, run_thread, (void *)closure) != 0) {
      printf("ERROR: Couldn't spawn thread %d\n", i);
    }
  }

  /* Wait for everything to complete */
  for (i=0; i<num_threads; i++) {
    pthread_join(tid[i], NULL);
  }

  for (i=0; i<num_threads; i++) {
    munmap((void *)buffers[i], map_sizes[i]);
    munmap((void *)anon_buffers[i], map_sizes[i]);
  }

  /* Record the longest single-call latency we saw */
  for (i=0; i<num_threads; i++) {
    if (mmap_time[i] > max_time) max_time = mmap_time[i];
  }

  printf ("Max MMAP Time: %llu usec\n", max_time);
  free(buffers);
  free(anon_buffers);
  free(directory);
  free(tid);
  free(map_sizes);
  return 0;
}