Makefile

all:
	gcc -g -Wall -O -o pktgen pktgen_stdout.c
	gcc -g -Wall -O -o ring ring.c

pktgen_stdout.c

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdint.h>
#include <string.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <net/ethernet.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#define __FAVOR_BSD
#include <netinet/udp.h>


#define PKTGEN_MAGIC   0xbe9be955
#define ETH_DST_MAC    0x020000000002
#define ETH_SRC_MAC    0x020000000001

#define IP4_SRC_IP     "10.0.0.1"
#define IP4_DST_IP     "10.0.0.2"
#define IP4_TTL        0x20
#define IP4_PROTO_UDP  0x11

#define UDP_SRC_PORT   0x09
#define UDP_DST_PORT   0x09

#define PKTDEV_HDR_LEN 12
#define ETH_HDR_LEN    14
#define IP4_HDR_LEN    20

#define PKTDEV_MAGIC   0x3776

/* from netmap pkt-gen.c */
static uint16_t checksum(const void * data, uint16_t len, uint32_t sum)
{
  const uint8_t *addr = data;
  uint32_t i;

  /* Checksum all the pairs of bytes first... */
  for (i = 0; i < (len & ~1U); i += 2) {
    sum += (u_int16_t)ntohs(*((u_int16_t *)(addr + i)));
    if (sum > 0xFFFF)
      sum -= 0xFFFF;
  }

  /*
   * If there's a single byte left over, checksum it, too.
   * Network byte order is big-endian, so the remaining byte is
   * the high byte.
   */
  if (i < len) {
    sum += addr[i] << 8;
    if (sum > 0xFFFF)
      sum -= 0xFFFF;
  }

  return sum;
}

static u_int16_t wrapsum(u_int32_t sum)
{
  sum = ~sum & 0xFFFF;
  return (htons(sum));
}


/* pktdev header */
struct pd_hdr {
	u_int16_t pd_magic;       /* le */
	u_int16_t pd_frame_len;   /* le */
	struct pd_timestamp {
		uint32_t val_low;
		uint16_t val_high;
		uint8_t resv2;
		uint8_t resv:4;
		uint8_t reg:3;
		bool reset:1;
	} __attribute__((packed)) pd_time;  /* le */
} __attribute__((packed));  /* le */

/* pktgen header */
struct pg_hdr {
  u_int32_t pg_magic;
  u_int32_t pg_id;
  u_int64_t pg_time;
} __attribute__((packed));

/* packet */
struct pktgen_pkt {
  struct pd_hdr pd;
  struct ether_header eth;
  struct ip ip;
  struct udphdr udp;
  struct pg_hdr pg;
} __attribute__((packed));



void set_pdhdr(struct pktgen_pkt *pkt, u_int16_t frame_len)
{
  struct pd_hdr *pd;
  pd = &pkt->pd;

  pd->pd_magic = PKTDEV_MAGIC;
  pd->pd_frame_len = frame_len;
  pd->pd_time.reset = 1;
  pd->pd_time.reg = 1;
  pd->pd_time.resv = 0;
  pd->pd_time.resv2 = 0;
  pd->pd_time.val_high = 0;
  pd->pd_time.val_low = 0;

  return;
}

void set_ethhdr(struct pktgen_pkt *pkt)
{
  struct ether_header *eth;
  eth = &pkt->eth;

  eth->ether_dhost[5] = (ETH_DST_MAC      ) & 0xFF;
  eth->ether_dhost[4] = (ETH_DST_MAC >>  8) & 0xFF;
  eth->ether_dhost[3] = (ETH_DST_MAC >> 16) & 0xFF;
  eth->ether_dhost[2] = (ETH_DST_MAC >> 24) & 0xFF;
  eth->ether_dhost[1] = (ETH_DST_MAC >> 32) & 0xFF;
  eth->ether_dhost[0] = (ETH_DST_MAC >> 40) & 0xFF;
  eth->ether_shost[5] = (ETH_SRC_MAC      ) & 0xFF;
  eth->ether_shost[4] = (ETH_SRC_MAC >>  8) & 0xFF;
  eth->ether_shost[3] = (ETH_SRC_MAC >> 16) & 0xFF;
  eth->ether_shost[2] = (ETH_SRC_MAC >> 24) & 0xFF;
  eth->ether_shost[1] = (ETH_SRC_MAC >> 32) & 0xFF;
  eth->ether_shost[0] = (ETH_SRC_MAC >> 40) & 0xFF;
  eth->ether_type = htons(ETHERTYPE_IP);

  return;
}

void set_ip4hdr(struct pktgen_pkt *pkt, u_int16_t frame_len)
{
  struct ip *ip;
  ip = &pkt->ip;

  ip->ip_v = IPVERSION;
  ip->ip_hl = 5;
  ip->ip_tos = 0;
  ip->ip_len = htons(frame_len - ETH_HDR_LEN);
  ip->ip_id = 0;
  ip->ip_off = htons (IP_DF);
  ip->ip_ttl = 0x20;
  ip->ip_p = IPPROTO_UDP;
  inet_pton(AF_INET, IP4_SRC_IP, &ip->ip_src);
  inet_pton(AF_INET, IP4_DST_IP, &ip->ip_dst);
  ip->ip_sum = 0;

  return;
}

void set_udphdr(struct pktgen_pkt *pkt, u_int16_t frame_len)
{
  struct udphdr *udp;
  udp = &pkt->udp;

  udp->uh_sport = htons(UDP_SRC_PORT);
  udp->uh_dport = htons(UDP_DST_PORT);
  udp->uh_ulen = htons(frame_len - ETH_HDR_LEN - IP4_HDR_LEN);
  udp->uh_sum = 0;

  return;
}

void set_pghdr(struct pktgen_pkt *pkt)
{
  struct pg_hdr *pg;
  pg = &pkt->pg;

  pg->pg_magic = htonl(PKTGEN_MAGIC);
  pg->pg_id = 0;
  pg->pg_time = 0;

  return;
};

//#define step 0xEE6B280
unsigned short id = 0;
unsigned long long ts = 0;
static inline void build_pack(char *pack, struct pktgen_pkt *pkt,
    unsigned int npkt, int pktlen, unsigned int step)
{
	struct ip *ip;
  int i, offset;

  ip = (struct ip *)&pkt->ip;

  offset = 0;
  for (i = 0; i < npkt; i++) {
    ip->ip_id = htons(id);
    pkt->pd.pd_time.val_low = ts & 0xFFFFFFFF;
    pkt->pd.pd_time.val_high = (ts >> 32) & 0xFFFF;
    pkt->pg.pg_id = htonl((u_int32_t)id++);
    ip->ip_sum = wrapsum(checksum(ip, sizeof(*ip), 0));
    //pkt->ip.ip_sum = 0;
    memcpy(pack + offset, pkt, sizeof(struct pktgen_pkt));
    pkt->pd.pd_time.reset = 0;
    offset += pktlen;
    ts += step;
  }
}

//#define mbps 1000
//#define step (int)(84 * (1000 / (float)mbps))
// ./pktgen_stdout -s <frame_len> -n <npkt> -m <nloop>
// ex(595 * 25010 = 14.88Mpps): ./pktgen_stdout -s 60 -n 595 -m 25010
int main(int argc, char **argv)
{
  char *pack = NULL;
  const char *ptr = NULL;
  struct pktgen_pkt *pkt = NULL;
  int ret = 0, i, pktlen, packlen, cnt, nleft;
  unsigned int step = 84;

  unsigned short frame_len = 60;
  unsigned int npkt = 5;
  unsigned int nloop = 10;
  unsigned int mbps = 1000;

  for (i = 1; i < argc; ++i) {
    if (0 == strcmp(argv[i], "-s")) {
      if (++i == argc) perror("-s");
      frame_len = atoi(argv[i]);
    } else if (0 == strcmp(argv[i], "-n")) {
      if (++i == argc) perror("-n");
      npkt = atoi(argv[i]);
    } else if (0 == strcmp(argv[i], "-m")) {
      if (++i == argc) perror("-m");
      nloop = atoi(argv[i]);
    } else if (0 == strcmp(argv[i], "-t")) {
      if (++i == argc) perror("-t");
      mbps = atoi(argv[i]);
    }
  }

  step = (unsigned int)(84 * (1000 / (float)mbps));
  fprintf(stderr, "step=%d\n", step);

  if (frame_len < 60 || frame_len > 9014) {
    fprintf(stderr, "frame size error: %d\n", frame_len);
    ret = -1;
    goto out;
  }
  if (npkt < 1) {
    fprintf(stderr, "npkt error: %d\n", (int)npkt);
    ret = -1;
    goto out;
  }
  if (nloop < 1) {
    fprintf(stderr, "nloop error: %d\n", nloop);
    ret = -1;
    goto out;
  }

  pkt = malloc(sizeof(struct pktgen_pkt));
  set_pdhdr(pkt, frame_len);
  set_ethhdr(pkt);
  set_ip4hdr(pkt, frame_len);
  set_udphdr(pkt, frame_len);
  set_pghdr(pkt);

  pktlen = PKTDEV_HDR_LEN + frame_len;
  pack = calloc((size_t)(pktlen * npkt), sizeof(char));

  // nloop
  packlen = pktlen * npkt;
  for (i = 0; i < nloop; i++) {
    nleft = packlen;
    ptr = (char *)pack;
    build_pack(pack, pkt, npkt, pktlen, step);
    while (nleft > 0) {
      if ((cnt = write(1, ptr, nleft)) <= 0) {
          // :todo (check errno)
          ;
      }
      nleft -= cnt;
      ptr += cnt;
    }
  }

out:
  if (pack)
    free(pack);

  if (pkt)
    free(pkt);

  return ret;
}

ring.c

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include <stdbool.h>
#include <string.h>
#include <libkern/OSByteOrder.h>

#define EP_XMIT_OK    0x10
#define EP_XMIT_BUSY  0x11
#define EP_XMIT_ERR   0x12
#define cpu_to_be16(x) OSSwapHostToBigInt16(x)
#define cpu_to_be64(x) OSSwapHostToBigInt64(x)
#define EP_MAGIC           0x3776
#define EP_HDR_SIZE        12       // magic:2 + frame_len:2 + ts:8
#define EP_HWHDR_SIZE      14       // frame_len:2 + hash:4 + ts:8
#define MAX_PKT_SIZE       9014
#define MIN_PKT_SIZE       40
#define RING_ALMOST_FULL   (MAX_PKT_SIZE*2)
#define XMIT_BUDGET        0x3F
#define pr_info printf
#define func_enter() pr_info("entering %s\n", __func__);
#define ALIGN(x,a)		__ALIGN_MASK(x,(typeof(x))(a)-1)
#define __ALIGN_MASK(x,mask)	(((x)+(mask))&~(mask))

struct ep_ring {
	uint32_t size;            /* malloc size of ring */
	uint8_t *start;           /* start address */
	uint8_t *end;             /* end address */
	uint32_t mask;            /* (size - 1) of ring */
	volatile uint8_t *read;   /* next position to be read */
	volatile uint8_t *write;  /* next position to be written */
};

struct ep_hw_pkt {
	uint16_t len;       /* frame length */
	uint64_t ts;        /* timestamp */
	uint32_t hash;      /* unused. reserved */
	uint8_t body[1];    /* ethnet frame data */
} __attribute__((__packed__));

struct mmio {
	uint8_t *virt;
	uint64_t start;
	uint64_t end;
	uint64_t flags;
	uint64_t len;
};

struct ecp3versa {
	struct pci_dev *pcidev;
	struct mmio mmio0;
	struct mmio mmio1;
	struct tx {
		volatile uint32_t *write;
		volatile uint32_t *read;
		uint32_t *start;
		uint32_t *end;
		uint32_t size;
		uint32_t mask;
	} tx;
};

struct ep_dev {
	struct ep_ring txq;
	struct ep_ring wrq;
	struct ecp3versa nic;
	struct ep_hw_pkt *hw_pkt;
	uint32_t tx_counter;
	uint32_t txq_size;
	uint32_t wrq_size;
};

struct ep_dev *pdev;
uint32_t txwr = 0;
uint32_t txrd = 0;

static inline bool ring_empty(const struct ep_ring *r)
{
	return !!(r->read == r->write);
}

static inline uint32_t ring_free_count(const struct ep_ring *r)
{
	return ((r->read - r->write - 1) & r->mask);
}

static inline bool ring_almost_full(const struct ep_ring *r)
{
	return !!(ring_free_count(r) < RING_ALMOST_FULL);
}

static inline void ring_read_next_aligned(struct ep_ring *r, uint32_t size)
{
	r->read += ALIGN(size, 4);
	if (r->read > r->end) {
		r->read = r->start;
	}
}

static inline uint16_t ring_next_magic(struct ep_ring *r)
{
	return *(uint16_t *)&r->read[0];
}

static inline uint16_t ring_next_frame_len(struct ep_ring *r)
{
	return *(uint16_t *)&r->read[2];
}
static inline uint64_t ring_next_timestamp(struct ep_ring *r)
{
	return *(uint64_t *)&r->read[4];
}
static inline uint32_t read_nic_txptr(uint32_t *p)
{
	return *p << 1;
}

static inline void set_nic_txptr(uint32_t *p, uint32_t addr)
{
	*p = addr >> 1;
}

static inline uint32_t hwtx_xmit_next(uint32_t hw_write, uint32_t size)
{
	struct ecp3versa *nic = &pdev->nic;

	hw_write += ALIGN(EP_HWHDR_SIZE + size, 2);
	hw_write &= nic->tx.mask;

	return hw_write;
}

static inline void ring_write_next(struct ep_ring *r, uint32_t size)
{
	r->write += size;
	if (r->write > r->end) {
		r->write = r->start;
	}
}

static inline void ring_read_next(struct ep_ring *r, uint32_t size)
{
	r->read += size;
	if (r->read > r->end) {
		r->read = r->start;
	}
}

static inline void ring_write_next_aligned(struct ep_ring *r, uint32_t size)
{
	r->write += ALIGN(size, 4);
	if (r->write > r->end) {
		r->write = r->start;
	}
}


/*
 * build_ep_pkt
 */
static inline int build_ep_pkt(struct ep_hw_pkt *pkt)
{
	uint16_t magic, frame_len;
	struct ep_ring *txq = &pdev->txq;

	// check magic code
	magic = ring_next_magic(txq);
	if (magic != EP_MAGIC) {
		pr_info("packet format error: magic=%X\n", (int)magic);
		return 0;
	}
	// check frame length
	frame_len = ring_next_frame_len(txq);
	if ((frame_len > MAX_PKT_SIZE) || (frame_len < MIN_PKT_SIZE)) {
		pr_info("packet format error: frame_len=%X\n", (int)frame_len);
		return 0;
	}

	pkt->len = cpu_to_be16(frame_len);
	pkt->hash = 0;
	pkt->ts = cpu_to_be64(ring_next_timestamp(txq));
	//pkt->ts = ring_next_timestamp(txq);

	memcpy(&pkt->body, (uint8_t *)(txq->read + EP_HDR_SIZE), frame_len);

	return frame_len;
}

/*
 * hxtx_alomost_full
 */
static inline bool hwtx_almost_full(uint32_t wr, uint32_t rd)
{
	return !!((rd - wr - 1) < 9000);
}

/*
 * xmit
 */
static inline void xmit(uint32_t wr, struct ep_hw_pkt *pkt, int len)
{
	uint8_t *nic_virt = pdev->nic.mmio1.virt;
	uint32_t tmp;

	len += EP_HWHDR_SIZE;

	if ((wr + len) < pdev->nic.tx.size) {
		memcpy(nic_virt + wr, pkt, len);
	} else {
		tmp = pdev->nic.tx.size - wr;
		//pr_info("overwriting: wr=%d, tmp=%d\n", wr, tmp);
		memcpy(nic_virt + wr, pkt, tmp);
		memcpy(nic_virt, ((uint8_t *)pkt + tmp), (len - tmp));
		//dump_nic_info((struct ep_hw_pkt *)((uint8_t *)pkt + tmp));
	}
}

/*
 * ethpipe_xmit
 */
static inline int ethpipe_xmit(uint32_t hw_write,
		uint32_t hw_read, int len)
{
	struct ep_ring *txq = &pdev->txq;
	int ret = EP_XMIT_OK;

	func_enter();

	if (!hwtx_almost_full(hw_write, hw_read)) {
		xmit(hw_write, pdev->hw_pkt, len);
		ring_read_next_aligned(txq, EP_HDR_SIZE + len);
		ret = EP_XMIT_OK;
	} else {
		ret = EP_XMIT_BUSY;
		pr_info("hwring is full\n");
	}

	return ret;
}

/*
 * ethpipe_send
 */
static inline void ethpipe_send(void)
{
	int limit, ret, len;
	uint32_t hw_write, hw_read;
	struct ep_ring *txq = &pdev->txq;

	func_enter();

	// read hwtx read and write address via pcie pio read
	hw_write = read_nic_txptr((uint32_t *)pdev->nic.tx.write);
	hw_read = read_nic_txptr((uint32_t *)pdev->nic.tx.read);

	// reset xmit budget
	limit = XMIT_BUDGET;

	// sending
	while(!ring_empty(txq) && (--limit > 0)) {
		len = build_ep_pkt(pdev->hw_pkt);
		if (len < 1) {
			pr_info("err: build_ep_pkt() len=%d\n", len);
			goto error;
		}

		ret = ethpipe_xmit(hw_write, hw_read, len);
		if (ret == EP_XMIT_OK) {
			hw_write = hwtx_xmit_next(hw_write, len);
			++pdev->tx_counter;    // incr tx_counter
		} else if (ret == EP_XMIT_BUSY) {
			goto out;
		} else {
			pr_info("err: unknown ret of ethpipe_xmit()\n");
			goto error;
		}
	}

	// commit to NIC via pcie pio write
	set_nic_txptr((uint32_t *)pdev->nic.tx.write, hw_write);

	// debug
	//dump_nic_info();

out:
	return;

error:
	pr_info("kthread: tx_err\n");
	// todo: need lock
	txq->read = txq->start;
	txq->write = txq->start;
	return;
}


void release(void)
{
	pr_info("%s\n", __func__);

	if(pdev->txq.start != NULL)
		free(pdev->txq.start);

	if(pdev->wrq.start != NULL)
		free(pdev->wrq.start);

	if(pdev->hw_pkt != NULL)
		free(pdev->hw_pkt);

	if(pdev->nic.mmio0.virt != NULL)
		free(pdev->nic.mmio0.virt);

	if(pdev->nic.mmio1.virt != NULL)
		free(pdev->nic.mmio1.virt);

	if(pdev != NULL)
		free(pdev);
}


void init(void)
{
	struct mmio *mmio0 = NULL;
	struct mmio *mmio1 = NULL;
	struct ecp3versa *nic = NULL;

	pr_info("%s\n", __func__);

	pdev = malloc(sizeof(struct ep_dev));
	pdev->tx_counter = 0;
	pdev->txq_size = 32 * 1024 * 1024;
	pdev->wrq_size = 32 * 1024 * 1024;
	pr_info("pdev->txq_size: %d\n", pdev->txq_size);
	pr_info("pdev->wrq_size: %d\n", pdev->wrq_size);

	/* temporary buffer for build paket */
	pdev->hw_pkt = (struct ep_hw_pkt *)malloc(
		sizeof(struct ep_hw_pkt) - sizeof(uint8_t) + (sizeof(uint8_t) * MAX_PKT_SIZE));
	if (pdev->hw_pkt == 0) {
		pr_info("fail to kmalloc: *pdev->hw_pkt\n");
		goto err;
	}

	/* setup transmit buffer */
	if ((pdev->txq.start =
			malloc(pdev->txq_size + EP_HDR_SIZE + MAX_PKT_SIZE)) == 0) {
		pr_info("fail to vmalloc: txq\n");
		goto err;
	}
	pdev->txq.size  = pdev->txq_size;
	pdev->txq.mask  = pdev->txq_size - 1;
	pdev->txq.end   = pdev->txq.start + pdev->txq_size - 1;
	pdev->txq.write = pdev->txq.start;
	pdev->txq.read  = pdev->txq.start;

	/* setup  */
	if ((pdev->wrq.start =
			malloc(pdev->wrq_size + EP_HDR_SIZE + MAX_PKT_SIZE)) == 0) {
		pr_info("fail to vmalloc: wrq\n");
		goto err;
	}
	pdev->wrq.size  = pdev->wrq_size;
	pdev->wrq.mask  = pdev->wrq_size - 1;
	pdev->wrq.end   = pdev->wrq.start + pdev->wrq_size - 1;
	pdev->wrq.write = pdev->wrq.start;
	pdev->wrq.read  = pdev->wrq.start;

	/* setup  */
	if ((pdev->nic.mmio0.virt = (uint8_t *)malloc(32 * 1024)) == 0) {
		pr_info("fail to vmalloc: mmio0\n");
		goto err;
	}
	mmio0 = &pdev->nic.mmio0;
	pr_info("mmio0->virt: %p\n", mmio0->virt);

	if ((pdev->nic.mmio1.virt = (uint8_t *)malloc(32 * 1024)) == 0) {
		pr_info("fail to vmalloc: mmio0\n");
		goto err;
	}
	mmio1 = &pdev->nic.mmio1;
	pr_info("mmio1->virt: %p\n", mmio1->virt);

	mmio1->len = 0x10000;

	nic = &pdev->nic;

	/* pointer of NIC registers */
	nic->tx.write = &txwr;
	nic->tx.read = &txrd;
	nic->tx.mask = nic->tx.size - 1;
	nic->tx.size = mmio1->len >> 1;
	pr_info("nic->tx.write: %p, %X\n", nic->tx.write, *nic->tx.write);
	pr_info("nic->tx.read: %p, %X\n", nic->tx.read, *nic->tx.read);
	pr_info("nic->tx.end: %p\n", nic->tx.end);
	pr_info("nic->tx.size: %X\n", (unsigned int)nic->tx.size);

	return;

err:
	release();
	exit(1);
}

void dump(void)
{
	struct ep_ring *t = &pdev->txq;

	printf("----------DUMP\n");
	printf("%02X %02X %02X %02X  ", t->read[ 0], t->read[ 1], t->read[ 2], t->read[ 3]);
	printf("%02X %02X %02X %02X\n", t->read[ 4], t->read[ 5], t->read[ 6], t->read[ 7]);

	printf("%02X %02X %02X %02X  ", t->read[ 8], t->read[ 9], t->read[10], t->read[11]);
	printf("%02X %02X %02X %02X\n", t->read[12], t->read[13], t->read[14], t->read[15]);
	printf("----------DUMP\n");
}

void work(void)
{
	struct ep_ring *wrq = &pdev->wrq;
	struct ep_ring *txq = &pdev->txq;
	int magic, frame_len, len;

	wrq->write = (volatile uint8_t *)wrq->start;
	wrq->read = (volatile uint8_t *)wrq->start;

	while(1){
		if (read(0, (uint8_t *)wrq->write, EP_HDR_SIZE) <= 0) {
			printf("No input data\n");
			dump();
			goto err;
		}
		ring_write_next(wrq, EP_HDR_SIZE);

		// check magic code
		magic = ring_next_magic(wrq);
		if (magic != EP_MAGIC) {
			pr_info("packet format error: magic=%X\n", (int)magic);
			goto err;
		}

		// check frame length
		frame_len = ring_next_frame_len(wrq);
		if ((frame_len > MAX_PKT_SIZE) || (frame_len < MIN_PKT_SIZE)) {
			pr_info("packet format error: frame_len=%X\n", (int)frame_len);
			goto err;
		} else {
			pr_info("frame_len=%X\n", (int)frame_len);
		}

		read(0, (uint8_t *)wrq->write, frame_len);
		ring_write_next(wrq, frame_len);

		len = EP_HDR_SIZE + frame_len;

		if (!ring_almost_full(txq)) {
			memcpy((uint8_t *)txq->write, (uint8_t *)wrq->read, len);
			ring_read_next(wrq, len);
			ring_write_next_aligned(txq, len);
		} else {
			// return when a ring buffer reached the max size
			pr_info("txq is full.\n");
			sleep(1);
			break;
		}

	}

err:
	return;
}


int main(void)
{
	init();

	work();
	ethpipe_send();

	release();

	return 0;
}