Doing fread on 128TB buffer

fread_128TiB.url

https://f.sldr.xyz/r/nMaMfjjWZs2WmDirit6Brx8Z

linker.ld

ENTRY(_start)
MEMORY
{
	/* Fun fact: first 64K of memory is not mapped by kernel at all */
	data (rwx) : ORIGIN = 64K, LENGTH = 64K
}
SECTIONS
{
	.text : { *(.text) } > data
	.data : { *(.data) } > data
	.bss : { *(.bss) } > data
}

main.c

// I do not want to mess with memory and use dynamically linked libc at the same time
#include "polyfills.c"  // so no libc today.

// We have 0x0000000000020000 to 0x00007fffffffffff of memory available
// which is almost 128TB!
// (see also: kernel.org/doc/Documentation/x86/x86_64/mm.txt)

// unfortuantely, we can't use it all :(
char* begin = (char*)0x200000;
char* end = (char*)0x7ff000000000;  // there is stack somewhere at the end, do not corrupt please
                                    // ASLR makes things much more difficult, by the way

void readFile();  // just a forward declaration
void _start() {
    // Idea is to use single 4GiB file (on tmpfs!)
    // and map it many times to fill whole memory region
    // Note: you may need to oncrease number of allowed mmap regions on your system
    i64 fd = open("./tmpfs/4GiB", O_RDWR, 0644);
    u64 size = 0x100000000; // trust me, it's 1GiB
    
    // Let's mmap it now
    char* base;
    for (base = begin; base < end; base += size) {
        void* ptr = mmap((u64)base,
                         size,
                         PROT_READ | PROT_WRITE,
                         MAP_SHARED | MAP_NORESERVE | MAP_FIXED_NOREPLACE,
                         fd,
                         0);
        i64 res = (i64)ptr;
        if (res < 0) {
            printStr("Stopped early\n");
            break;
        }
    }
    end = base;
    
    // OK, we now have really large buffer.
    u64 allocated = end - begin;
    u64 tb = allocated / 1024/1024/1024/1024;  // Convert to TB
    printStr("Allocated: "); printNum(tb); printStr("TB\n");
    
    // Next part is doing fread() using that buffer as destination.
    // I'll be very surprised if kernel is really going to read all 128TiB in one take
    printStr("Reading /dev/zero\n");
    i32 src = open("/dev/zero", O_RDONLY, 0);
    readFile(src);
    
    // Then compare with plain 4GiB file to make sure that /dev/zero is no special
    // (ba dum tsss, /dev/zero is implemented as «_special_ character device»)
    printStr("Reading ./tmpfs/4GiB\n");
    src = open("./tmpfs/4GiB", O_RDONLY, 0);
    readFile(src);
    
    // you know, _start should not return normally. There is nowhere to return.
    exit(0);
}

void readFile(int src) {
    char* dst = begin;
    for (int i = 0; i < 3; ++i) {
        u64 allocated = end - dst;
        i64 res = read(src, dst, allocated);
        printStr("    "); printNum(res);
        printStr(" out of "); printNum(allocated);
        printStr("\n");
        if (res > 0) dst += res;
        if (res < 0) break;
    }
}

run.sh

sudo mount -t tmpfs tmpfs ./tmpfs
fallocate -l 4G ./tmpfs/4GiB

sudo sysctl -w vm.max_map_count=6553000

gcc -g -O2 -nostdlib -c ./main.c
ld ./main.o -T linker.ld -o fread_128TB

./fread_128TB