ZFS ARC poisoning in FreeBSD 11.2

zfsarc.c

// clang -g -Wall -O2 -o zfsarc zfsarc.c

#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/types.h>

#define ITERATIONS 1000000


void randreads(const char *file, struct stat *sb)
{
	FILE *fd;
	char buffer[256 * 1024];
	struct timeval start;

	gettimeofday(&start, NULL);

	if ((fd = fopen(file, "r")) == NULL)
	{
		fprintf(stderr, "Couldn't open %s: %s\n", file, strerror(errno));
		exit(1);
	}

	for (int i = 1; i <= ITERATIONS; i++)
	{
		uint32_t bytestoread = arc4random_uniform(sizeof(buffer));
		uint32_t position = arc4random_uniform(sb->st_size - sizeof(buffer));

		bytestoread = bytestoread ? bytestoread : 1;

		if (fseek(fd, position, SEEK_SET) != 0)
		{
			fprintf(stderr, "Could't seek to %i in %s: %s\n", position, file, strerror(errno));
			fclose(fd);
			exit(1);
		}

		if (fread(buffer, bytestoread, 1, fd) != 1)
		{
			fprintf(stderr, "Couldn't read %i bytes from %s: %s\n", bytestoread, file, strerror(errno));
			fclose(fd);
			exit(1);
		}

		if (!(i % 100000))
		{
			struct timeval now;

			gettimeofday(&now, NULL);
            printf(" - %i: %li Reads per second\n", i, i / (now.tv_sec - start.tv_sec)); 
		}
	}

	fclose(fd);
}

void fullread(const char *file, struct stat *sb)
{
	FILE *fd;
	char buffer[256 * 1024];
	const uint32_t iterations = sb->st_size / sizeof(buffer);

	if ((fd = fopen(file, "r")) == NULL)
	{
		fprintf(stderr, "Couldn't open %s: %s\n", file, strerror(errno));
		exit(1);
	}

	for (int i = 1; i <= iterations; i++)
	{
		uint32_t bytestoread = (i == iterations) ? sb->st_size % sizeof(buffer) : sizeof(buffer);

		if (bytestoread > 0) 
		{
			if (fread(buffer, bytestoread, 1, fd) != 1)
			{
				fprintf(stderr, "Couldn't read %i bytes from %s: %s\n", bytestoread, file, strerror(errno));
				fclose(fd);
				exit(1);
			}
		}
	}

	fclose(fd);
}

int main(int argc, char *argv[])
{
	// No stdout buffering.
	setbuf(stdout, NULL);

	// Get arguments.
	if (argc != 3)
	{
		fprintf(stderr, "Usage: ./test file_a file_b\n");
		exit(1);
	}

	const char *file_a = argv[1];
	const char *file_b = argv[2];

	
	// Check if the files are large enough.
	// We need at least the size of the ARC.
	uint64_t arcmax;
	size_t arglen = sizeof(arcmax);

	if (sysctlbyname("vfs.zfs.arc_max", &arcmax, &arglen, NULL, 0) != 0)
	{
		fprintf(stderr, "Couldn't get vfs.zfs.arc_max: %s\n", strerror(errno));
		exit(1);
	}
	

	struct stat sb_a;

    if (stat(file_a, &sb_a) != 0)
	{
		fprintf(stderr, "Couldn't stat %s: %s\n", file_a, strerror(errno));
		exit(1);
	}

	if (!S_ISREG(sb_a.st_mode))
	{
		fprintf(stderr, "%s is not a file.\n", file_a);
		exit(1);
	}

	if (sb_a.st_size < arcmax)
	{
		fprintf(stderr, "%s is to small: Must be at least %lu bytes\n", file_a, arcmax);
		exit(1);
	}


	struct stat sb_b;

    if (stat(file_b, &sb_b) != 0)
	{
		fprintf(stderr, "Couldn't stat %s: %s\n", file_b, strerror(errno));
		exit(1);
	}

	if (!S_ISREG(sb_b.st_mode))
	{
		fprintf(stderr, "%s is not a file.\n", file_b);
		exit(1);
	}

	if (sb_b.st_size < arcmax)
	{
		fprintf(stderr, "%s is to small: Must be at least %lu bytes\n", file_b, arcmax);
		exit(1);
	}


	// Do ITERATIONS random reads on file A. This will fill up the ARC.
	// In theory this should start slow and get faster as the ARC fills
	// up.
	printf("1. Doing %i random reads from %s:\n", ITERATIONS, file_a);
	randreads(file_a, &sb_a);

	// Poison the ARC by reading once through file B. This will write
	// contents from file B into the ARC, replacing contents from file
	// A.
	printf("2. Reading %s from start to end.\n", file_b);
	fullread(file_b, &sb_b);

	// Do ITERATIONS random reads on file A. This should start slow and
	// get faster as contents of file B is replaced by contents of file
	// A. This can be seen in 11.1-RELEASE, but not in 11.2-RELEASE.
	printf("3. Doing %i random reads from %s:\n", ITERATIONS, file_a);
	randreads(file_a, &sb_a);

	// Shrink the ARC by 1 byte to alter it's configuration.
	printf("4. Shrinking the ARC by one byte.\n");

	arcmax--;

	if (sysctlbyname("vfs.zfs.arc_max", NULL, NULL, &arcmax, arglen) != 0)
	{
		fprintf(stderr, "Couldn't set vfs.zfs.arc_max: %s\n", strerror(errno));
		exit(1);
	}

	// Do ITERATIONS random reads on file A. In 11.2-RELEASE this will show
	// good performance, again.
	printf("5. Doing %i random reads from %s:\n", ITERATIONS, file_a);
	randreads(file_a, &sb_a);

	return 0;
}