Overlay/"hole punch" a section of an `mmap`ed region with another `mmap`ed region

main.c

#include <bits/time.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

int main() {
  const size_t page_size = 4096 * 64;
  const size_t total_size = page_size * 1024;

  int baseFd = open("base.bin", O_RDONLY);
  if (baseFd == -1) {
    perror("Could not open base file");

    return EXIT_FAILURE;
  }

  int overlayFd = open("overlay.bin", O_RDONLY);
  if (overlayFd == -1) {
    perror("Could not open overlay file");

    return EXIT_FAILURE;
  }

  struct timespec before, after;

  if (clock_gettime(CLOCK_MONOTONIC, &before) == -1) {
    perror("Could not measure time before");

    close(baseFd);

    return EXIT_FAILURE;
  }

  char *baseMap = mmap(NULL, total_size, PROT_READ, MAP_PRIVATE, baseFd, 0);
  if (baseMap == MAP_FAILED) {
    perror("Could not map base file");

    close(baseFd);

    return EXIT_FAILURE;
  }

  if (clock_gettime(CLOCK_MONOTONIC, &after) == -1) {
    perror("Could not measure time after");

    close(baseFd);

    return EXIT_FAILURE;
  }

  printf("mmap(\"base.bin\") took %lins\n", after.tv_nsec - before.tv_nsec);

  if (clock_gettime(CLOCK_MONOTONIC, &before) == -1) {
    perror("Could not measure time before");

    close(baseFd);

    return EXIT_FAILURE;
  }

  int pages_added = 0;
  for (size_t offset = 0; offset < total_size; offset += page_size * 2) {
    char *overlayMap = mmap(baseMap + offset, page_size, PROT_READ,
                            MAP_PRIVATE | MAP_FIXED, overlayFd, offset);
    if (overlayMap == MAP_FAILED) {
      perror("Could not map overlay file");

      close(baseFd);
      close(overlayFd);

      return EXIT_FAILURE;
    }

    pages_added++;
  }

  if (clock_gettime(CLOCK_MONOTONIC, &after) == -1) {
    perror("Could not measure time after");

    close(baseFd);
    close(overlayFd);

    return EXIT_FAILURE;
  }

  printf("mmap(\"overlay.bin\") (%i pages) took %lins\n", pages_added,
         after.tv_nsec - before.tv_nsec);

  // Sanity check
  printf("8 chars of first page: \"%.8s\"\n", baseMap + (page_size * 0));
  printf("8 chars of second page: \"%.8s\"\n", baseMap + (page_size * 1));
  printf("8 chars of third page: \"%.8s\"\n", baseMap + (page_size * 2));
  printf("8 chars of fourth page: \"%.8s\"\n", baseMap + (page_size * 3));

  return EXIT_SUCCESS;
}

main.go

package main

import (
	"io"
	"os"
)

func main() {
	pageSize := 4096 * 64
	totalSize := pageSize * 1024

	{
		out, err := os.OpenFile("base.bin", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
		if err != nil {
			panic(err)
		}
		defer out.Close()

		if err := out.Truncate(int64(totalSize)); err != nil {
			panic(err)
		}
	}

	{
		out, err := os.OpenFile("overlay.bin", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
		if err != nil {
			panic(err)
		}
		defer out.Close()

		in, err := os.Open("/dev/random")
		if err != nil {
			panic(err)
		}
		defer in.Close()

		if _, err := io.CopyN(out, in, int64(totalSize)); err != nil {
			panic(err)
		}
	}
}

main_threaded.c

#include <bits/time.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

#define NUM_THREADS 20

typedef struct {
  char *baseMap;
  int overlayFd;
  size_t start_offset;
  size_t end_offset;
  size_t page_size;
} thread_arg;

void *map_pages(void *arg) {
  thread_arg *data = (thread_arg *)arg;

  int pages_added = 0;
  for (size_t offset = data->start_offset; offset < data->end_offset;
       offset += data->page_size * 2) {
    char *overlayMap = mmap(data->baseMap + offset, data->page_size, PROT_READ,
                            MAP_SHARED | MAP_FIXED, data->overlayFd, offset);
    if (overlayMap == MAP_FAILED) {
      perror("Could not map overlay file");

      continue;
    }

    pages_added++;
  }

  return (void *)(intptr_t)pages_added;
}

int main() {
  const size_t page_size = 4096 * 64;
  const size_t total_size = page_size * 1024 * 10;

  int baseFd = open("base.bin", O_RDONLY);
  if (baseFd == -1) {
    perror("Could not open base file");

    return EXIT_FAILURE;
  }

  int overlayFd = open("overlay.bin", O_RDONLY);
  if (overlayFd == -1) {
    perror("Could not open overlay file");

    return EXIT_FAILURE;
  }

  struct timespec before, after;

  if (clock_gettime(CLOCK_MONOTONIC, &before) == -1) {
    perror("Could not measure time before");

    close(baseFd);

    return EXIT_FAILURE;
  }

  char *baseMap = mmap(NULL, total_size, PROT_READ, MAP_PRIVATE, baseFd, 0);
  if (baseMap == MAP_FAILED) {
    perror("Could not map base file");

    close(baseFd);

    return EXIT_FAILURE;
  }

  if (clock_gettime(CLOCK_MONOTONIC, &after) == -1) {
    perror("Could not measure time after");

    close(baseFd);

    return EXIT_FAILURE;
  }

  printf("mmap(\"base.bin\") took %lins\n", after.tv_nsec - before.tv_nsec);

  if (clock_gettime(CLOCK_MONOTONIC, &before) == -1) {
    perror("Could not measure time before");

    close(baseFd);

    return EXIT_FAILURE;
  }

  pthread_t threads[NUM_THREADS];
  thread_arg args[NUM_THREADS];

  size_t step = (total_size + NUM_THREADS - 1) / NUM_THREADS;
  step = (step + page_size - 1) / (page_size * 2) *
         (page_size * 2); // Make sure step is a multiple of page_size * 2

  for (int i = 0; i < NUM_THREADS; i++) {
    size_t start_offset = i * step;
    size_t end_offset = start_offset + step;
    if (start_offset >= total_size)
      break;
    if (end_offset > total_size)
      end_offset = total_size;

    args[i].baseMap = baseMap;
    args[i].overlayFd = overlayFd;
    args[i].start_offset = start_offset;
    args[i].end_offset = end_offset;
    args[i].page_size = page_size;

    if (pthread_create(&threads[i], NULL, map_pages, &args[i]) != 0) {
      perror("Could not create thread");

      close(baseFd);
      close(overlayFd);

      return EXIT_FAILURE;
    }
  }

  int pages_added = 0;
  for (int i = 0; i < NUM_THREADS; i++) {
    void *status;
    if (pthread_join(threads[i], &status) == 0) {
      pages_added += (intptr_t)status;
    }
  }

  if (clock_gettime(CLOCK_MONOTONIC, &after) == -1) {
    perror("Could not measure time after");

    close(baseFd);
    close(overlayFd);

    return EXIT_FAILURE;
  }

  //   // Sanity check
  //   for (int i = 0; i < pages_added; i++) {
  //     printf("8 chars of page %i: \"%.8s\"\n", i + 1, baseMap + (page_size *
  //     i));
  //   }

  printf("mmap(\"overlay.bin\") (%i pages using %i threads) took %lins\n",
         pages_added, NUM_THREADS, after.tv_nsec - before.tv_nsec);

  return EXIT_SUCCESS;
}

main_threaded.go

package main

import (
	"io"
	"log"
	"os"
)

func main() {
	pageSize := 4096 * 64
	totalSize := pageSize * 1024 * 10

	{
		log.Println("Writing base file")

		out, err := os.OpenFile("base.bin", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
		if err != nil {
			panic(err)
		}
		defer out.Close()

		if err := out.Truncate(int64(totalSize)); err != nil {
			panic(err)
		}
	}

	{
		log.Println("Writing overlay file")

		out, err := os.OpenFile("overlay.bin", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
		if err != nil {
			panic(err)
		}
		defer out.Close()

		in, err := os.Open("/dev/random")
		if err != nil {
			panic(err)
		}
		defer in.Close()

		if _, err := io.CopyN(out, in, int64(totalSize)); err != nil {
			panic(err)
		}
	}
}