load_store.c

#include <stdio.h>
#include <stddef.h>
#include <syscall.h>
#include <pthread.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <x86intrin.h>

typedef unsigned long u64;
typedef volatile u64 foo_t __attribute__ ((aligned(64)));
foo_t no_notify = 0, used = 0, avail = 0, kick = 0, n_kicks = 0, n_kicks_skipped = 0;

static void __attribute__((noinline))
delay(u64 d)
{
  u64 t0 = _rdtsc();
  while (_rdtsc() - t0 < d)
    _mm_pause();
}

void *
rx_thread_fn (void *arg)
{
  unsigned cpu, node;
  u64 my_avail = 0;
  syscall (__NR_getcpu, &cpu, &node, 0);
  printf ("rx thread running on cpu %u node %u\n", cpu, node);

  while (1)
    {
      while (__atomic_exchange_n (&kick, 0, __ATOMIC_RELAXED)  == 0)
	{
	  _mm_pause();
	}
      n_kicks++;

next:
     my_avail = avail;
     _mm_lfence();
     no_notify = 1;
     delay (20);
     used = my_avail;
     no_notify = 0;
     _mm_mfence();
     if (avail != my_avail)
       goto next;

    }
  return 0;
}

void *
tx_thread_fn (void *arg)
{
  unsigned cpu, node;
  syscall (__NR_getcpu, &cpu, &node, 0);
  printf ("tx thread running on cpu %u node %u\n", cpu, node);
  while (1)
    {
      u64 n_avail = 256 + used - avail;

      if (n_avail == 0)
	continue;

      delay(50);
#if 1
      avail += n_avail;
      //_mm_mfence();
      //_mm_pause();
      if (no_notify == 0)
#else
      __atomic_store_n (&avail, avail + n_avail, __ATOMIC_SEQ_CST);
     if (__atomic_load_n (&no_notify, __ATOMIC_SEQ_CST)  == 0)
#endif
	kick = 1;
      else
	n_kicks_skipped++;
    }
  return 0;
}

int
main (int argc, char **argv)
{
  int err;
  pthread_t rxt, txt;

  if ((err = pthread_create (&rxt, NULL, &rx_thread_fn, NULL)))
    {
      printf ("\ncan't create thread :[%s]", strerror (err));
      exit (1);
    }

  if ((err = pthread_create (&txt, NULL, &tx_thread_fn, NULL)))
    {
      printf ("\ncan't create thread :[%s]", strerror (err));
      exit (1);
    }

  while (1)
    {
      sleep (1);
      printf ("avail %lu used %lu n_kicks %lu n_kicks_skipped %lu\n",
	       avail, used, n_kicks, n_kicks_skipped);
    }
}