memory mapped buffer in C

memory_buffer.c

#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>

static int file_exists(const char* path) {
	struct stat buffer;
	return stat(path, &buffer) == 0;
}

static char* str_dupl(const char* src) {
	size_t src_size = strlen(src) + 1;
	char* newstr = malloc(src_size);
	memcpy(newstr, src, src_size);
	return newstr;
}

static const size_t ROT_MAX_NUMBER_SPACE = 50;

struct filerot {
	const char* patt;
	char* buf;
	size_t size;
	size_t count;
};

void filerot_init(struct filerot* fr, const char* patt, size_t size) {
	assert(size > ROT_MAX_NUMBER_SPACE);
	fr->patt = patt;
	fr->buf = calloc(1, size);
	fr->size = size;
	fr->count = 0;
}

void filerot_print(struct filerot* fr) {
	snprintf(fr->buf, fr->size, "%s_%zu_", fr->patt, fr->count);
	++(fr->count);
}

void filerot_deinit(struct filerot* fr) {
	free(fr->buf);
}

static const int32_t DEFAULT_START_SIZE = 10;
static const size_t FILE_HEADER_SIZE = sizeof(int32_t) * 2;


struct mapped {
	char* path;
	int fd;
	unsigned char* data;
};

int32_t mapped_len(struct mapped* mpd) {
	return ((int32_t*)mpd->data)[0];
}

int32_t mapped_cap(struct mapped* mpd) {
	return ((int32_t*)mpd->data)[1];
}

int32_t mapped_file_size(struct mapped* mpd) {
	return mapped_cap(mpd) + (sizeof(int32_t) * 2);
}

void mapped_cap_set(struct mapped* mpd, int32_t num) {
	((int32_t*)mpd->data)[1] = num;
}

void mapped_len_inc(struct mapped* mpd, int32_t amount) {
	((int32_t*)mpd->data)[0] += amount;
}

void mapped_len_dec(struct mapped* mpd, int32_t amount) {
	((int32_t*)mpd->data)[0] -= amount;
}

unsigned char* mapped_data_start(struct mapped* mpd) {
	return mpd->data + (sizeof(int32_t) * 2);
}

int mapped_open(struct mapped* mpd, const char* path) {
	// todo ftruncate ?
	struct stat buffer;
	int was_existing = 0;
	memset(mpd, 0, sizeof(*mpd));
	if (stat(path, &buffer) == 0) {
		int fd = open(path, O_RDWR | O_CREAT, (mode_t)0600);
		if (fd == -1) {
			return -1;
		}
		was_existing = 1;
		//mpd->fd = fd;
		int32_t file_info[2] = {0};
		ssize_t size_bytes = read(fd, &file_info, sizeof(file_info));
		if (size_bytes != sizeof(file_info)) {
			close(fd);
			// error case
			return -1;
		}
		lseek(fd, 0, SEEK_SET);
		unsigned char* map = mmap(0, file_info[1] + FILE_HEADER_SIZE , PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
		mpd->fd = fd;
		mpd->path = str_dupl(path);
		mpd->data = map;
	} else {
		int fd = open(path, O_RDWR | O_CREAT, (mode_t)0600);
		int32_t total_size = FILE_HEADER_SIZE + DEFAULT_START_SIZE;
		lseek(fd, total_size-1, SEEK_SET);
		write(fd, "", 1);
		lseek(fd, 0, SEEK_SET);
		unsigned char* map = mmap(0, total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
		mpd->fd = fd;
		mpd->path = str_dupl(path);
		mpd->data = map;
		mapped_cap_set(mpd, DEFAULT_START_SIZE);
	}
	return was_existing;
}

int mapped_grow(struct mapped* mpd, int32_t new_size) {
	int32_t cur_size = mapped_file_size(mpd);
	//printf("cur size is %u, new size is %u\n", cur_size, new_size);
	if (cur_size >= new_size || new_size < 0) {
		return 0;
	}
	msync(mpd->data, cur_size, MS_SYNC); // needed?
	munmap(mpd->data, cur_size);
	lseek(mpd->fd, new_size-1, SEEK_SET);
	write(mpd->fd, "", 1);
	lseek(mpd->fd, 0, SEEK_SET);
	unsigned char* map = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, mpd->fd, 0); //todo NULL ?
	mpd->data = map;
	mapped_cap_set(mpd, new_size);
	return 1;
}

int mapped_can_append(struct mapped* mpd, int32_t size) {
	int32_t fsize_now = mapped_file_size(mpd);
	return (fsize_now + size) >= fsize_now;
}

int32_t mapped_append(struct mapped* mpd, void* buf, int32_t size) {
	int32_t cur_size = mapped_cap(mpd);
	int32_t cur_len = mapped_len(mpd);
	//printf("appending size %d, cur len is %u, cur cap is %u\n", size, cur_len, cur_size);
	if ((cur_len + size) > cur_size) {
		if(!mapped_grow(mpd, cur_size + size + (cur_size * 2))) {
			return -1;
		}
	}
	memcpy(mapped_data_start(mpd) + cur_len, buf, size);
	mapped_len_inc(mpd, size);
	return cur_len; // offset of last item appended
}

int mapped_write(struct mapped* mpd, int32_t offset, void* buf, int32_t size) {
	int32_t cur_cap = mapped_cap(mpd);
	if ((offset + size) >= cur_cap) {
		return 0;
	}
	memcpy(mapped_data_start(mpd) + offset, buf, size);
	return 1;
}

int32_t mapped_append_null_bytes(struct mapped* mpd, int32_t size) {
	int32_t cur_len = mapped_len(mpd);
	int32_t cur_size = mapped_cap(mpd);
	if ((cur_len + size) > cur_size) {
		if(!mapped_grow(mpd, cur_size + size + (cur_size * 2))) {
			return -1;
		}
	}
	memset(mapped_data_start(mpd) + cur_len, 0, size);
	mapped_len_inc(mpd, size);
	return cur_len;
}

int32_t mapped_remove(struct mapped* mpd, int32_t offset, int32_t size) {
	int32_t cur_cap = mapped_cap(mpd);
	if ((offset + size) >= cur_cap) {
		return -1;
	}
	memset(mapped_data_start(mpd) + offset, 0, size);
	return offset;
}

int mapped_erase(struct mapped* mpd, int32_t offset, int32_t amount) {
	int32_t cur_cap = mapped_cap(mpd);
	if ((offset + amount) >= cur_cap) {
		return 0;
	}
	unsigned char* dat_start = mapped_data_start(mpd);
	int32_t mem_end = offset + amount;
	int32_t mem_cpy_size = cur_cap - mem_end;
	unsigned char* cpy_buf = malloc(mem_cpy_size);
	memcpy(cpy_buf, dat_start + mem_end, amount);
	memcpy(dat_start + offset, cpy_buf, amount);
	mapped_len_dec(mpd, amount);
	free(cpy_buf);
	return 1;
}

int mapped_pop(struct mapped* mpd, void* buf, int32_t size) {
	int32_t last_len = mapped_len(mpd) - size;
	unsigned char* last_offset = mapped_data_start(mpd) + last_len;
	memcpy(buf, last_offset, size);
	mapped_len_dec(mpd, size);
	return 1;
}

int mapped_sync(struct mapped* mpd) {
	msync(mpd->data, mapped_file_size(mpd), MS_SYNC);
	return 1;
}

int mapped_close(struct mapped* mpd) {
	msync(mpd->data, mapped_file_size(mpd), MS_SYNC);
	munmap(mpd->data, mapped_file_size(mpd));
	close(mpd->fd);
	mpd->fd = -1;
	return 1;
}

void mapped_path_free(struct mapped* mpd) {
	free(mpd->path);
	mpd->path = NULL;
}

int mapped_path_remove(struct mapped* mpd) {
	if (mpd->path != NULL) {
		return remove(mpd->path);
	} else {
		return -1;
	}
}

void mapped_print(struct mapped* mpd) {
	unsigned char* mapd = mapped_data_start(mpd);
	int32_t cur_len = mapped_len(mpd);
	int32_t cur_cap = mapped_cap(mpd);
	printf("cur len is %u, cur cap is %u\n", cur_len, cur_cap);
	for (int j = 0; j < cur_len; ++j)
	{
		printf("%u, \n", mapd[j]);
	}
}

static const char* MAP_STORE_BLOCK_EXT = "_blok";
static const char* MAP_STORE_IDX_EXT = "_idx";
static const char* MAP_STORE_SPACES_EXT = "_spaces";

// location format is 
// [index in block][size in block]
struct mapped_store {
	struct mapped block;
	struct mapped indexes;
	struct mapped spaces;
	//todo free space total?
};

int mapped_store_open(struct mapped_store* mst, const char* basepath) {
	char pathbuf[1024] = {'\0'};
	strcpy(pathbuf, basepath);
	strcat(pathbuf, MAP_STORE_BLOCK_EXT);
	mapped_open(&mst->block, pathbuf);

	strcpy(pathbuf, basepath);
	strcat(pathbuf, MAP_STORE_IDX_EXT);
	mapped_open(&mst->indexes, pathbuf);

	strcpy(pathbuf, basepath);
	strcat(pathbuf, MAP_STORE_SPACES_EXT);
	mapped_open(&mst->spaces, pathbuf);

	return 1;
}

int mapped_store_describe_index(struct mapped_store* mst, int32_t alloc_index, int32_t* result) {
	if (alloc_index >= mapped_len(&mst->indexes))
		return 0;
	int32_t* index_data = (int32_t*)mapped_data_start(&mst->indexes);
	result[0] = index_data[alloc_index];
	result[1] = index_data[alloc_index + 1];
	return 1;
}

int mapped_store_describe_space(struct mapped_store* mst, int32_t space_index, int32_t* result) {
	if (space_index >= mapped_len(&mst->spaces))
		return 0;
	int32_t* space_data = (int32_t*)mapped_data_start(&mst->spaces);
	result[0] = space_data[space_index];
	result[1] = space_data[space_index + 1];
	return 1;
}

int32_t mapped_store_item_len(struct mapped_store* mst) { // todo size_t ?
	return mapped_len(&mst->indexes) / (2 * sizeof(int32_t));
}

int mapped_store_find_space(struct mapped_store* mst, int32_t amount, int32_t* result) {
	if (mapped_len(&mst->spaces) == 0) {
		return 0;
	}
	int32_t* space_data = (int32_t*)mapped_data_start(&mst->spaces);
	int32_t space_len = mapped_len(&mst->spaces);
	for (int32_t i = 0; i < space_len; i += 2) {
		if (space_data[i+1] >= amount) {
			int32_t index_write[2] = {0};
			int32_t old_space_size = space_data[i+1];
			int32_t old_space_index = space_data[i];
			int32_t new_space_size = space_data[i+1] - amount;
			int32_t new_space_index = space_data[i] + amount;
			index_write[0] = new_space_index;
			index_write[1] = new_space_size;
			int32_t alloc_index = mapped_append(&mst->indexes, index_write, sizeof(index_write)) / sizeof(int32_t);
			*result = alloc_index;

			space_data[i] = new_space_index;
			space_data[i+1] = new_space_size;
			if (new_space_size == 0) {
				// todo consider in memory list of these
				mapped_erase(&mst->spaces, i, 2 * sizeof(int32_t));
			}
			return 1;
		}
	}
	return 0;
}

// index format is [block index][block amount]
void mapped_store_create_space(struct mapped_store* mst, int32_t amount, int32_t* result) {
	int32_t index_write[2] = {0};
	int32_t alloc_block = mapped_append_null_bytes(&mst->block, amount);
	index_write[0] = alloc_block;
	index_write[1] = amount;
	int32_t alloc_index = mapped_append(&mst->indexes, index_write, sizeof(index_write)) / sizeof(int32_t);
	*result = alloc_index;
}

int mapped_store_allocate(struct mapped_store* mst, int32_t amount, int32_t* result) {
	if (mapped_len(&mst->spaces) == 0) {
		mapped_store_create_space(mst, amount, result);
	} else { // todo check max size deleted in the future.
		if (!mapped_store_find_space(mst, amount, result)) {
			mapped_store_create_space(mst, amount, result);
		}
	}
	return 1;
}

int mapped_store_deallocate(struct mapped_store* mst, int32_t index_n, int32_t* result) {
	int32_t desc[2] = {-1};
	if(!mapped_store_describe_index(mst, index_n, desc)) {
		return 0;
	}
	int32_t erase_begin = index_n * sizeof(int32_t);
	int32_t erase_amnt = (2 * sizeof(int32_t));
	mapped_erase(&mst->indexes, erase_begin ,  erase_amnt); //todo async erase?
	int32_t space_index = mapped_append(&mst->spaces, desc, sizeof(desc)) / sizeof(int32_t);
	*result = space_index;
	return 1;
}

// todo support compaction

int mapped_store_close(struct mapped_store* mst) {
	mapped_close(&mst->block);
	mapped_close(&mst->indexes);
	mapped_close(&mst->spaces);
	return 1;
}

void mapped_store_path_free(struct mapped_store* mst) {
	mapped_path_free(&mst->block);
	mapped_path_free(&mst->indexes);
	mapped_path_free(&mst->spaces);
}

int mapped_store_remove_files(struct mapped_store* mst) {
	int result1, result2, result3;
	result1 = mapped_path_remove(&mst->block);
	result2 = mapped_path_remove(&mst->indexes);
	result3 = mapped_path_remove(&mst->spaces);
	return result1 != -1 && result2 != -1 && result3 != -1;
}

static void check_cond(int cond, const char* condstr, unsigned line) {
	if (!cond) {
		fprintf(stderr, "Failed cond '%s' at line %u\n", condstr, line);
	}
}

#define CHECKIT(cnd) check_cond(cnd, #cnd, __LINE__)

static void test_mapped_append_grow(void) {
	const char* testfile = "foo";
	struct mapped md;
	mapped_open(&md, testfile);
	uint32_t cur_len = mapped_len(&md);
	uint32_t cur_cap = mapped_cap(&md);
	for (unsigned char i = 0; i < DEFAULT_START_SIZE + 6; ++i)
	{
		mapped_append(&md, &i, sizeof(i));
	}
	CHECKIT(mapped_len(&md) > cur_len);
	CHECKIT(mapped_cap(&md) > cur_cap);
	mapped_close(&md);
	mapped_path_remove(&md);
	mapped_path_free(&md);
}

static void test_mapped_append_erase(void) {
	const char* testfile = "foo";
	struct mapped md;
	mapped_open(&md, testfile);
	unsigned char ch1 = 30;
	int32_t append_off1 = mapped_append(&md, &ch1, sizeof(ch1));
	unsigned char ch2 = 40;
	int32_t append_off2 = mapped_append(&md, &ch2, sizeof(ch2));
	unsigned char ch3 = 50;
	int32_t append_off3 = mapped_append(&md, &ch3, sizeof(ch3));
	CHECKIT(mapped_len(&md) == 3);
	mapped_erase(&md, 0, 2);
	CHECKIT(mapped_len(&md) == 1);
	CHECKIT(mapped_data_start(&md)[0] == 50);
	mapped_close(&md);
	remove(testfile);
}

static void test_mapped_append_null(void) {
	const char* testfile = "foo";
	struct mapped md;
	mapped_open(&md, testfile);
	CHECKIT(mapped_len(&md) == 0);
	mapped_append_null_bytes(&md, 100);
	CHECKIT(mapped_len(&md) == 100);
	CHECKIT(mapped_data_start(&md)[0] == 0);
	CHECKIT(mapped_data_start(&md)[99] == 0);
	mapped_close(&md);
	remove(testfile);
}

static void test_mapped_disk_persist(void) {
	const char* testfile = "ddfoo";
	struct mapped md;
	struct mapped md2;
	int32_t cap_mark = -1;
	mapped_open(&md, testfile);
	CHECKIT(mapped_len(&md) == 0);
	mapped_append_null_bytes(&md, 100);
	CHECKIT(mapped_len(&md) == 100);
	CHECKIT(mapped_data_start(&md)[0] == 0);
	CHECKIT(mapped_data_start(&md)[99] == 0);
	cap_mark = mapped_cap(&md);
	mapped_close(&md);
	mapped_open(&md2, testfile);
	CHECKIT(mapped_len(&md2) == 100);
	CHECKIT(mapped_data_start(&md2)[0] == 0);
	CHECKIT(mapped_data_start(&md2)[99] == 0);
	CHECKIT(mapped_cap(&md2) == cap_mark);
	mapped_close(&md2);
	remove(testfile);
}

static void test_mapped_store_open_close(void) {
	const char* testpattern = "foobar";
	struct mapped_store mds;
	CHECKIT(mapped_store_open(&mds, testpattern));
	CHECKIT(mapped_store_close(&mds));
	CHECKIT(mapped_store_remove_files(&mds));
	mapped_store_path_free(&mds);
}

static void test_mapped_store_allocate(void) {
	const char* testpattern = "foobar";
	struct mapped_store mds;
	int32_t ind_res = -1;
	int32_t desc[2] = {-1};
	CHECKIT(mapped_store_open(&mds, testpattern));
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(ind_res == 0);
	CHECKIT(mapped_len(&(mds.block)) == 8);
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(ind_res == 2 );
	CHECKIT(mapped_len(&(mds.block)) == 16);
	CHECKIT(mapped_len(&(mds.indexes)) == 4 * sizeof(int32_t));
	CHECKIT(mapped_store_describe_index(&mds, ind_res, desc));
	CHECKIT(desc[0] == 8);
	CHECKIT(desc[1] == 8);
	CHECKIT(mapped_store_close(&mds));
	CHECKIT(mapped_store_remove_files(&mds));
	mapped_store_path_free(&mds);
}

static void test_mapped_store_deallocate(void) {
	const char* testpattern = "foobar";
	struct mapped_store mds;
	int32_t ind_res = -1;
	int32_t dealo_res = -1;
	int32_t desc[2] = {-1};
	CHECKIT(mapped_store_open(&mds, testpattern));
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(ind_res == 0);
	CHECKIT(mapped_len(&(mds.block)) == 8);
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(mapped_store_item_len(&mds) == 2);
	CHECKIT(ind_res == 2 );
	CHECKIT(mapped_len(&(mds.block)) == 16);
	CHECKIT(mapped_len(&(mds.indexes)) == 4 * sizeof(int32_t));
	//printf("%d\n", mapped_len(&(mds.indexes)));
	CHECKIT(mapped_store_deallocate(&mds, ind_res, &dealo_res));
	//printf("%d\n", mapped_len(&(mds.indexes)));
	CHECKIT(mapped_len(&(mds.indexes)) == 2 * sizeof(int32_t));
	CHECKIT(mapped_len(&(mds.spaces)) == 2 * sizeof(int32_t));
	CHECKIT(dealo_res == 0);
	CHECKIT(mapped_store_close(&mds));
	CHECKIT(mapped_store_remove_files(&mds));
	mapped_store_path_free(&mds);
}

static void test_mapped_store_reallocate(void) {
	const char* testpattern = "foobar";
	struct mapped_store mds;
	int32_t ind_res = -1;
	int32_t dealo_res = -1;
	int32_t desc[2] = {-1};
	int32_t space_desc[2] = {-1};
	CHECKIT(mapped_store_open(&mds, testpattern));
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(ind_res == 0);
	CHECKIT(mapped_len(&(mds.block)) == 8);
	CHECKIT(mapped_store_allocate(&mds, 8, &ind_res));
	CHECKIT(ind_res == 2 );
	CHECKIT(mapped_len(&(mds.block)) == 16);
	CHECKIT(mapped_len(&(mds.indexes)) == 4 * sizeof(int32_t));
	CHECKIT(mapped_len(&(mds.spaces)) == 0);
	CHECKIT(mapped_store_deallocate(&mds, ind_res, &dealo_res));
	CHECKIT(mapped_len(&(mds.indexes)) == 2 * sizeof(int32_t));
	CHECKIT(dealo_res == 0);
	ind_res = -1;
	CHECKIT(mapped_store_allocate(&mds, 4, &ind_res)); // reallocation
	CHECKIT(mapped_store_describe_space(&mds, dealo_res, space_desc));
	CHECKIT(ind_res == 2);
	CHECKIT(mapped_len(&(mds.spaces)) == 2 * sizeof(int32_t));
	CHECKIT(mapped_store_allocate(&mds, 4, &ind_res)); // reallocation

	CHECKIT(mapped_len(&(mds.spaces)) == 0); // empty delete blocks should be gone
	CHECKIT(ind_res == 4);
	CHECKIT(mapped_store_close(&mds));
	CHECKIT(mapped_store_remove_files(&mds));
	mapped_store_path_free(&mds);
}

int main(int argc, char const *argv[])
{
	test_mapped_append_grow();
	test_mapped_append_erase();
	test_mapped_append_null();
	test_mapped_disk_persist();
	test_mapped_store_open_close();
	test_mapped_store_allocate();
	test_mapped_store_deallocate();
	test_mapped_store_reallocate();
	return 0;
}