Ugly hack to redirect all malloc calls to a mmap(2)'ed file.

userspaceswap.c

/*
 * Redirect malloc(3) calls to a mmap(2)'ed file.
 *
 * Example of usage:
 * $ cc -shared -o userspaceswap.dylib ./userspaceswap.c && \
 *   DYLD_FORCE_FLAT_NAMESPACE=1 DYLD_INSERT_LIBRARIES=./userspaceswap.dylib ffmpeg
 */

#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>

#define MALLOC_MAP_HEAP_SIZE (1<<30)
#define MALLOC_MAP_SCRATCH_FILE "scratch_space.mmap"

void *_malloc_mmap_p;

static void _malloc_mmap_init(void)
{
    int fd = open(MALLOC_MAP_SCRATCH_FILE, O_RDWR|O_TRUNC|O_CREAT, 0770);
    if (fd == -1) {
        //fprintf(stderr, "INIT: %d: %s\n", errno, strerror(errno));
        exit(1);
    }
    ftruncate(fd, MALLOC_MAP_HEAP_SIZE);
    _malloc_mmap_p = mmap(NULL, MALLOC_MAP_HEAP_SIZE-1, PROT_WRITE|PROT_EXEC|PROT_READ, MAP_SHARED, fd, 0);
    if (_malloc_mmap_p == NULL) {
        exit(1);
    }
    //fprintf(stderr, "FAKE MALLOC INIT: %p\n", _malloc_mmap_p);
}

void
*malloc(size_t size)
{
    //fprintf(stderr, "malloc(%zu) = ", size);
    if (size == 0)
        return NULL;
    if ((size%8) != 0) {
        size += 8 - size % 8;
    }

    // XXX NON ATOMIC INIT
    if (_malloc_mmap_p == NULL) {
        _malloc_mmap_init();
    }

    void *p = (void *)__sync_fetch_and_add((unsigned long long*)&_malloc_mmap_p, (unsigned long long)size);
    //fprintf(stderr, "%p:%zu\n", p, size);
    return p;
}

void
*realloc(void *ptr, size_t size)
{
    //fprintf(stderr, "realloc(%p, %zu) = ", ptr, size);
    if (size == 0) {
        return NULL;
    }
    if (ptr == NULL) {
        return malloc(size);
    }

    // XXX NON ATOMIC INIT
    if (_malloc_mmap_p == NULL) {
        _malloc_mmap_init();
    }

    // XXX NON ATOMIC INCR
    void *p = malloc(size);
    memmove(p, ptr, size);

    return p;
}

void
*reallocf(void *ptr, size_t size)
{
    //fprintf(stderr, "reallocf(%zu) = ", size);

    // XXX NON ATOMIC INIT
    if (_malloc_mmap_p == NULL) {
        _malloc_mmap_init();
    }
    return realloc(ptr, size);
}

void
*valloc(size_t size)
{
    // XXX ALIGN
    exit(1);
}

void
*calloc(size_t count, size_t size)
{
    //fprintf(stderr, "calloc(%zu, %zu) = ", count, size);

    // XXX NON ATOMIC INIT
    if (_malloc_mmap_p == NULL) {
        _malloc_mmap_init();
    }

    // XXX OVERFLOW
    void *p = malloc(size*count);
    //fprintf(stderr, "%p\n", p);
    return p;
}

void
free(void *p)
{
    if (_malloc_mmap_p == NULL) {
        _malloc_mmap_init();
    }
}