experiments on memory consistency (x86)

dekker.cpp

// compile: g++ dekker.cpp -lpthread

#include <stdio.h>
#include <thread>
#include <unistd.h>
#include <pthread.h>
#include <sys/syscall.h>

#define N               100000
#define USE_MEMFENCE    1
#define CACHE_LINE_SIZE 64

struct Data {
  char pad0[CACHE_LINE_SIZE];
  volatile int A = 0;
  char pad1[CACHE_LINE_SIZE];
  volatile int B = 0;
  char pad2[CACHE_LINE_SIZE];
};

volatile int ready = 0;
pthread_barrier_t ready_barrier, finish_barrier;
volatile int A_result, B_result;
Data data;

pid_t gettid(void) {
  return syscall(SYS_gettid);
}

int set_affinity(int cpu) {
  pid_t pid = gettid();
  cpu_set_t cpuset;
  CPU_ZERO(&cpuset);
  CPU_SET(cpu, &cpuset);
  int result = sched_setaffinity(pid, sizeof(cpu_set_t), &cpuset);
  if (result != 0 || sched_getcpu() != cpu) {
    fprintf(stderr, "failed to set affinity.\n");
    exit(1);
  }
}

int main() {
  int result[2][2] = {{0, 0}, {0, 0}};

  pthread_barrier_init(&ready_barrier , NULL, 3);
  pthread_barrier_init(&finish_barrier, NULL, 3);

  set_affinity(0);

  auto thread1 = std::thread([] {
    set_affinity(1);
    for (int i = 0; i < N; i++) {
      pthread_barrier_wait(&ready_barrier);
      while (!ready);

      data.A = 1;
#if USE_MEMFENCE
      asm volatile("mfence":::"memory");
#endif
      B_result = data.B;

      pthread_barrier_wait(&finish_barrier);
    }
  });
  auto thread2 = std::thread([] {
    set_affinity(2);
    for (int i = 0; i < N; i++) {
      pthread_barrier_wait(&ready_barrier);
      while (!ready);

      data.B = 1;
#if USE_MEMFENCE
      asm volatile("mfence":::"memory");
#endif
      A_result = data.A;

      pthread_barrier_wait(&finish_barrier);
    }
  });

  for (int i = 0; i < N; i++) {
    pthread_barrier_wait(&ready_barrier);
    ready = 1;
    pthread_barrier_wait(&finish_barrier);
    result[A_result][B_result]++;
    data.A = 0; data.B = 0;
    ready = 0;
  }

  thread1.join();
  thread2.join();

  for (int a = 0; a < 2; a++)
    for (int b = 0; b < 2; b++)
      printf("(A == %d, B == %d) : %d\n", a, b, result[a][b]);

  return 0;
}