thp bloat

thp_usage.c

#define _GNU_SOURCE

#include <error.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>

#define REGION_SIZE (128UL * 1024 * 1024 * 1024)

char waitkey() {
	fprintf(stdout, "waiting for key ...\n");
	return getchar();
}

int main()
{
	fprintf(stdout, "PID %d\n", getpid());
	waitkey();
	
	int prot = PROT_READ | PROT_WRITE;
	int flags = MAP_ANONYMOUS | MAP_PRIVATE;
	char *buf = mmap(NULL, REGION_SIZE, prot, flags, -1, 0);
	if (buf == MAP_FAILED) {
	  int err = errno;
	  fprintf(stderr, "Error allocating buffer: %s\n", strerror(err));
	  return err;
	}
	fprintf(stdout, "Allocated buffer at %p\n", buf);

	int page_size = getpagesize();
	unsigned long npages = REGION_SIZE / page_size;

	fprintf(stdout, "Before MADV_HUGEPAGE\n");
	int ret = madvise(buf, REGION_SIZE, MADV_HUGEPAGE);
	if (ret == -1) {
	  int err = errno;
	  fprintf(stderr, "madvise MADV_HUGEPAGE failed\n");
	  return err;
	}
	fprintf(stdout, "After MADV_HUGEPAGE\n");

	fprintf(stdout, "Entering write/madvise(dontneed) loop ...\n");
	while (1) {
		unsigned long idx = rand() % npages;
		double r = (double)rand() / (double)RAND_MAX;
		if (r > 0.3) {
			buf[idx * page_size] = 'x';
		} else {
			ret = madvise(buf + (idx * page_size), page_size, MADV_DONTNEED);
			if (ret == -1) {
				int err = errno;
				fprintf(stderr, "madvise MADV_DONTNEED failed\n");
				return err;
			}
		}
	}


	fprintf(stdout, "Before mnumap\n");
	munmap(buf, REGION_SIZE);
	fprintf(stdout, "After munmap\n");
	
	return 0;
}

thp_usage.png

thp_usage.txt

Memory usage of a test program which mmaps 128G area and writes to a
random address in a loop. Together with writes, madvise(MADV_DONTNEED)
is issued at another random address. Writes are issued with 70%
probability and DONTNEED 30%. With this test, I'm trying to emulate
workload of a large in memory hash-table.


The figure compares memory usage with and without the thp-bloat patch.