fsync.c

disk.py

import os
import time
import random
import shutil

def test_write_performance(file_count, file_size_mb):
    """
    Tests the performance of writing to files.

    Args:
        file_count (int): Number of files to create.
        file_size_mb (int): Size of each file in megabytes.
    """
    os.system("sudo sysctl -w vm.drop_caches=3")

    # Create test files
    test_files = []
    for i in range(file_count):
        file_name = f"test_file_{i}.txt"
        test_files.append(file_name)

    # Measure write time
    write_times = []
    write_time = time.time()
    for file_name in test_files:
        with open(file_name, "w") as f:
            f.write("x" * (file_size_mb * 1024 * 1024))
    write_time = time.time() - write_time
    write_times.append(write_time)

    # Calculate average write time
    average_write_time = sum(write_times) / len(write_times)

    print(f"Average write time: {average_write_time:.5f} seconds")

def test_read_performance(file_count, file_size_mb):
    """
    Tests the performance of reading from files.

    Args:
        file_count (int): Number of files to create.
        file_size_mb (int): Size of each file in megabytes.
    """

    # Clean page cache
    os.system("sudo sysctl -w vm.drop_caches=3")

    # Measure read time
    read_times = []
    test_files = []
    for i in range(file_count):
        file_name = f"test_file_{i}.txt"
        test_files.append(file_name)

    read_time = time.time()
    for file_name in test_files:
        with open(file_name, "r") as f:
            f.read()
    read_time = time.time() - read_time
    read_times.append(read_time)

    # Calculate average read time
    average_read_time = sum(read_times) / len(read_times)

    print(f"Average read time: {average_read_time:.5f} seconds")


test_write_performance(1, 2000)
os.remove("test_file_0.txt")
test_write_performance(10, 200)
test_read_performance(1, 2000)
test_read_performance(10, 200)

fsync.c

#define _GNU_SOURCE
#include <fcntl.h>
#include <liburing.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
//------- UTILS  -------
#define FILE_NUMBER 300
#define BYTES_NUMBER 200
#define RUN_TIMES 10
#define QUEUE_DEPTH 640
#define KB (1024)
#define MB (1024 * 1024)
#define GB (1024 * 1024 * 1024)

char Buffer[BYTES_NUMBER];

int
clr_pgcache() {
  return system("echo 1 > /proc/sys/vm/drop_caches");
}

int *
create_files(int n, int flags) {
  int *fds = (int *)malloc(n * sizeof(int));
  for (int i = 0; i < n; i++) {
    char filename[15];
    if (sprintf(filename, "file%d", i) < 0) {
      printf("Failed to create filename\n");
      free(fds);
      return NULL;
    }

    int fd = open(filename, flags, 0666);
    if (fd == -1) {
      printf("Failed to open file\n");
      free(fds);
      return NULL;
    }
    fds[i] = fd;
  }
  return fds;
}

int
close_files(int *fd, int n) {
  char filename[15];
  for (int i = 0; i < n; i++) {
    if (close(fd[i]) == -1) {
      printf("Failed to close file\n");
      free(fd);
      return -1;
    }

    // remove files
    if (sprintf(filename, "file%d", i) < 0) {
      printf("Failed to create filename\n");
      free(fd);
      return -1;
    }
    if (remove(filename) == -1) {
      printf("Failed to remove file\n");
      free(fd);
      return -1;
    }
  }
  free(fd);
  return 0;
}

int
setup_io_uring(struct io_uring *ring) {
  if (io_uring_queue_init(QUEUE_DEPTH, ring, 0) < 0) {
    printf("io_uring_queue_init");
    return -1;
  }
  return 0;
}

void
submit_write_request(struct io_uring *ring, int fd, int offset, char *buffer,
                     int buffer_size) {
  struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
  if (sqe == NULL) {
    printf("io_uring_get_sqe");
    return;
  }
  io_uring_prep_write(sqe, fd, buffer, buffer_size, offset);
  io_uring_sqe_set_flags(sqe, 0);
}

void
submit_fsync_request(struct io_uring *ring, int fd) {
  struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
  if (sqe == NULL) {
    printf("io_uring_get_sqe");
    return;
  }
  io_uring_prep_fsync(sqe, fd, 0);
  io_uring_sqe_set_flags(sqe, 0);
}

void
wait_for_all_operations(struct io_uring *ring, int n) {
  struct io_uring_cqe *cqe;
  int completed = 0;
  while (completed < n) {
    io_uring_submit_and_wait(ring, 1);
    while (io_uring_peek_cqe(ring, &cqe) == 0) {
      if (cqe->res < 0) {
        printf("Error: %d\n", cqe->res);
        return;
      }
      io_uring_cqe_seen(ring, cqe);
      completed++;
    }
  }
}

// define marco to measure time
#define MEASURE_TIME_START() \
  struct timeval start, end; \
  gettimeofday(&start, NULL);

#define MEASURE_TIME_END() gettimeofday(&end, NULL);

#define MEASURE_TIME_US() \
  (end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec

#define MEASURE_TIME_MS() \
  (end.tv_sec - start.tv_sec) * 1000 + (end.tv_usec - start.tv_usec) / 1000

//------- Benchmark ------
void
seq_write_speed() {
  clr_pgcache();
  int fd = open("file", O_RDWR | O_CREAT, 0666);
  if (fd == -1) {
    printf("Failed to open file\n");
    return;
  }

  MEASURE_TIME_START();
  // sequence write 4K to file 1000 times
  for (int i = 0; i < 1000000; i++) {
    if (write(fd, Buffer, BYTES_NUMBER) != BYTES_NUMBER) {
      printf("Failed to write to file\n");
      return;
    }
  }
  MEASURE_TIME_END();
  if (close(fd) == -1) {
    printf("Failed to close file\n");
    return;
  }
  if (remove("file") == -1) {
    printf("Failed to remove file\n");
    return;
  }
  unsigned long total_bytes = (unsigned long)1000000 * BYTES_NUMBER;
  unsigned long cost_ms = MEASURE_TIME_MS();
  unsigned long cost_s = cost_ms / 1000;
  printf("Sequential write speed: %f MB/s\n",
         (float)total_bytes / MB / cost_ms * 1000);
}

void
fsync_speed() {
  clr_pgcache();
  int fd = open("file", O_RDWR | O_CREAT, 0666);
  if (fd == -1) {
    printf("Failed to open file\n");
    return;
  }

  MEASURE_TIME_START();
  for (int i = 0; i < 10000; i++) {
    if (fsync(fd) == -1) {
      printf("Failed to fsync file\n");
      return;
    }
  }
  MEASURE_TIME_END();
  if (close(fd) == -1) {
    printf("Failed to close file\n");
    return;
  }
  if (remove("file") == -1) {
    printf("Failed to remove file\n");
    return;
  }
  unsigned long cost_ms = MEASURE_TIME_MS();
  printf("fsync avg time: %ld ms\n", cost_ms);
}

//-------Scenarios-----
int scenario1();

int scenario2();

int scenario3();

int scenario4();

int scenario5();

int scenario6();

int scenario7();

int scenario8();

int
main(int argc, char *argv[]) {
  memset(Buffer, 'a', BYTES_NUMBER);
  printf("Do Base Line Test\n");
  seq_write_speed();
  fsync_speed();
  printf("---- Scenario 1: Open Write Close ----\n");
  int cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario1();
    printf("Run %d: %d us\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 1\n");
      return -1;
    }
  }
  printf("Average: %d us\n", cost_sum / RUN_TIMES);
  printf(
      "scenario1 write speed: %f MB/s\n",
      (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000000 / MB);

  printf("---- Scenario 2: Open Write Fsync Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario2();
    printf("Run %d: %d ms\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 2\n");
      return -1;
    }
  }
  printf("Average: %d ms\n", cost_sum / RUN_TIMES);
  printf("scenario2 write speed: %f MB/s\n",
         (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000 / MB);

  printf("---- Scenario 3: Open Write(DIRECT) Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario3();
    printf("Run %d: %d us\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 3\n");
      return -1;
    }
  }
  printf("Average: %d us\n", cost_sum / RUN_TIMES);
  printf(
      "scenario3 write speed: %f MB/s\n",
      (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000000 / MB);

  printf("---- Scenario 4: Open Write(DIRECT) Fsync Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario4();
    printf("Run %d: %d ms\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 4\n");
      return -1;
    }
  }
  printf("Average: %d ms\n", cost_sum / RUN_TIMES);
  printf("scenario4 write speed: %f MB/s\n",
         (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000 / MB);

  printf("---- Scenario 5: (Single File) Open Write Fsync Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario5();
    printf("Run %d: %d us\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 5\n");
      return -1;
    }
  }
  printf("Average: %d us\n", cost_sum / RUN_TIMES);
  printf(
      "scenario5 write speed: %f MB/s\n",
      (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000000 / MB);

  printf("---- Scenario 6: (io_uring) Open Write Fsync Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario6();
    printf("Run %d: %d ms\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 6\n");
      return -1;
    }
  }
  printf("Average: %d ms\n", cost_sum / RUN_TIMES);
  printf("scenario6 write speed: %f MB/s\n",
         (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000 / MB);

  printf(
      "---- Scenario 7: (io_uring Single File) Open Write with Fsync Close "
      "----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario7();
    printf("Run %d: %d ms\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 7\n");
      return -1;
    }
  }
  printf("Average: %d ms\n", cost_sum / RUN_TIMES);
  printf("scenario7 write speed: %f MB/s\n",
         (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000 / MB);

  printf(
      "---- Scenario 8: (io_uring Single File) Open Write Fsync Close ----\n");
  cost_sum = 0;
  for (int i = 0; i < RUN_TIMES; i++) {
    int cost = scenario8();
    printf("Run %d: %d us\n", i, cost);
    cost_sum += cost;
    if (cost == -1) {
      printf("Failed to run scenario 8\n");
      return -1;
    }
  }
  printf("Average: %d ms\n", cost_sum / RUN_TIMES);
  printf(
      "scenario8 write speed: %f MB/s\n",
      (float)FILE_NUMBER * BYTES_NUMBER * RUN_TIMES / cost_sum * 1000000 / MB);
}

int
scenario1() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  // open many files, store the file descriptor in an array
  int *fd = create_files(FILE_NUMBER, O_RDWR | O_CREAT);

  MEASURE_TIME_START();
  // each file write some bytes
  for (int i = 0; i < FILE_NUMBER; i++) {
    if (write(fd[i], Buffer, 200) != 200) {
      printf("Failed to write to file\n");
      return -1;
    }
  }
  MEASURE_TIME_END();

  if (close_files(fd, FILE_NUMBER) != 0) {
    printf("Failed to close files\n");
    return -1;
  }

  return MEASURE_TIME_US();
}

int
scenario2() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  // open many files, store the file descriptor in an array
  int *fd = create_files(FILE_NUMBER, O_RDWR | O_CREAT);

  MEASURE_TIME_START();
  // each file write some bytes
  for (int i = 0; i < FILE_NUMBER; i++) {
    if (write(fd[i], Buffer, 200) != 200) {
      printf("Failed to write to file\n");
      return -1;
    }
    if (fsync(fd[i]) == -1) {
      printf("Failed to fsync file\n");
      return -1;
    }
  }
  MEASURE_TIME_END();

  if (close_files(fd, FILE_NUMBER) != 0) {
    printf("Failed to close files\n");
    return -1;
  }

  return MEASURE_TIME_MS();
}

int
scenario3() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  // open many files, store the file descriptor in an array
  int *fd = create_files(FILE_NUMBER, O_RDWR | O_CREAT | O_DIRECT);

  MEASURE_TIME_START();
  // each file write some bytes
  for (int i = 0; i < FILE_NUMBER; i++) {
    if (write(fd[i], Buffer, 200) != 200) {
      printf("Failed to write to file\n");
      return -1;
    }
  }
  MEASURE_TIME_END();

  if (close_files(fd, FILE_NUMBER) != 0) {
    printf("Failed to close files\n");
    return -1;
  }

  return MEASURE_TIME_US();
}

int
scenario4() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  // open many files, store the file descriptor in an array
  int *fd = create_files(FILE_NUMBER, O_RDWR | O_CREAT | O_DIRECT);

  MEASURE_TIME_START();
  // each file write some bytes
  for (int i = 0; i < FILE_NUMBER; i++) {
    if (write(fd[i], Buffer, 200) != 200) {
      printf("Failed to write to file\n");
      return -1;
    }
    if (fsync(fd[i]) == -1) {
      printf("Failed to fsync file\n");
      return -1;
    }
  }
  MEASURE_TIME_END();

  if (close_files(fd, FILE_NUMBER) != 0) {
    printf("Failed to close files\n");
    return -1;
  }

  return MEASURE_TIME_MS();
}

int
scenario5() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  int fd = open("file", O_RDWR | O_CREAT, 0666);
  if (fd == -1) {
    printf("Failed to open file\n");
    return -1;
  }

  MEASURE_TIME_START();
  // write some BYTES_NUMBER
  for (int i = 0; i < FILE_NUMBER; i++) {
    if (write(fd, Buffer, BYTES_NUMBER) != BYTES_NUMBER) {
      printf("Failed to write to file\n");
      return -1;
    }
  }
  if (fsync(fd) == -1) {
    printf("Failed to fsync file\n");
    return -1;
  }

  MEASURE_TIME_END();
  if (close(fd) == -1) {
    printf("Failed to close file\n");
    return -1;
  }
  if (remove("file") == -1) {
    printf("Failed to remove file\n");
    return -1;
  }
  return MEASURE_TIME_US();
}

int
scenario6() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  int *fd = create_files(FILE_NUMBER, O_RDWR | O_CREAT);

  struct io_uring ring;
  if (setup_io_uring(&ring) != 0) {
    printf("Failed to setup io_uring\n");
    return -1;
  }

  MEASURE_TIME_START();
  for (int i = 0; i < FILE_NUMBER; i++) {
    submit_write_request(&ring, fd[i], 0, Buffer, BYTES_NUMBER);
    submit_fsync_request(&ring, fd[i]);
  }
  wait_for_all_operations(&ring, 2 * FILE_NUMBER);
  MEASURE_TIME_END();
  close_files(fd, FILE_NUMBER);
  return MEASURE_TIME_MS();
}

int
scenario7() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  int fd = open("file", O_RDWR | O_CREAT, 0666);
  if (fd == -1) {
    printf("Failed to open file\n");
    return -1;
  }
  struct io_uring ring;
  if (setup_io_uring(&ring) != 0) {
    printf("Failed to setup io_uring\n");
    return -1;
  }

  MEASURE_TIME_START();
  unsigned offset = 0;
  for (int i = 0; i < FILE_NUMBER; i++) {
    submit_write_request(&ring, fd, offset, Buffer, BYTES_NUMBER);
    submit_fsync_request(&ring, fd);
    offset += BYTES_NUMBER;
  }
  wait_for_all_operations(&ring, 2 * FILE_NUMBER);
  MEASURE_TIME_END();
  if (close(fd) == -1) {
    printf("Failed to close file\n");
    return -1;
  }
  if (remove("file") == -1) {
    printf("Failed to remove file\n");
    return -1;
  }
  return MEASURE_TIME_MS();
}

int
scenario8() {
  if (clr_pgcache() != 0) {
    printf("Failed to clear page cache\n");
    return -1;
  }

  int fd = open("file", O_RDWR | O_CREAT, 0666);
  if (fd == -1) {
    printf("Failed to open file\n");
    return -1;
  }
  struct io_uring ring;
  if (setup_io_uring(&ring) != 0) {
    printf("Failed to setup io_uring\n");
    return -1;
  }

  MEASURE_TIME_START();
  unsigned offset = 0;
  for (int i = 0; i < FILE_NUMBER; i++) {
    submit_write_request(&ring, fd, offset, Buffer, BYTES_NUMBER);
    offset += BYTES_NUMBER;
  }
  submit_fsync_request(&ring, fd);
  wait_for_all_operations(&ring, FILE_NUMBER + 1);
  MEASURE_TIME_END();
  if (close(fd) == -1) {
    printf("Failed to close file\n");
    return -1;
  }
  if (remove("file") == -1) {
    printf("Failed to remove file\n");
    return -1;
  }
  return MEASURE_TIME_US();
}

result

Do Base Line Test
Sequential write speed: 199.304977 MB/s
fsync avg time: 2 ms
---- Scenario 1: Open Write Close ----
Run 0: 430 us
Run 1: 424 us
Run 2: 449 us
Run 3: 428 us
Run 4: 402 us
Run 5: 406 us
Run 6: 432 us
Run 7: 451 us
Run 8: 440 us
Run 9: 445 us
Average: 430 us
scenario1 write speed: 132.854568 MB/s
---- Scenario 2: Open Write Fsync Close ----
Run 0: 155 ms
Run 1: 156 ms
Run 2: 157 ms
Run 3: 149 ms
Run 4: 150 ms
Run 5: 156 ms
Run 6: 150 ms
Run 7: 154 ms
Run 8: 156 ms
Run 9: 159 ms
Average: 154 ms
scenario2 write speed: 0.371080 MB/s
---- Scenario 3: Open Write(DIRECT) Close ----
Run 0: 4672 us
Run 1: 4338 us
Run 2: 3640 us
Run 3: 4242 us
Run 4: 3539 us
Run 5: 3759 us
Run 6: 4274 us
Run 7: 4638 us
Run 8: 4872 us
Run 9: 3980 us
Average: 4195 us
scenario3 write speed: 13.638856 MB/s
---- Scenario 4: Open Write(DIRECT) Fsync Close ----
Run 0: 179 ms
Run 1: 155 ms
Run 2: 147 ms
Run 3: 150 ms
Run 4: 152 ms
Run 5: 150 ms
Run 6: 152 ms
Run 7: 154 ms
Run 8: 151 ms
Run 9: 149 ms
Average: 153 ms
scenario4 write speed: 0.371803 MB/s
---- Scenario 5: (Single File) Open Write Fsync Close ----
Run 0: 1123 us
Run 1: 1034 us
Run 2: 1021 us
Run 3: 1178 us
Run 4: 1033 us
Run 5: 1140 us
Run 6: 970 us
Run 7: 999 us
Run 8: 948 us
Run 9: 1056 us
Average: 1050 us
scenario5 write speed: 54.485294 MB/s
---- Scenario 6: (io_uring) Open Write Fsync Close ----
Run 0: 9 ms
Run 1: 19 ms
Run 2: 24 ms
Run 3: 11 ms
Run 4: 26 ms
Run 5: 42 ms
Run 6: 78 ms
Run 7: 23 ms
Run 8: 37 ms
Run 9: 33 ms
Average: 30 ms
scenario6 write speed: 1.894717 MB/s
---- Scenario 7: (io_uring Single File) Open Write with Fsync Close ----
Run 0: 2 ms
Run 1: 3 ms
Run 2: 2 ms
Run 3: 2 ms
Run 4: 2 ms
Run 5: 3 ms
Run 6: 2 ms
Run 7: 2 ms
Run 8: 2 ms
Run 9: 2 ms
Average: 2 ms
scenario7 write speed: 26.009298 MB/s
---- Scenario 8: (io_uring Single File) Open Write Fsync Close ----
Run 0: 869 us
Run 1: 884 us
Run 2: 950 us
Run 3: 831 us
Run 4: 825 us
Run 5: 611 us
Run 6: 877 us
Run 7: 911 us
Run 8: 906 us
Run 9: 806 us
Average: 847 ms
scenario8 write speed: 67.556618 MB/s