echo latency test for gvisor

.clang-format

BasedOnStyle: LLVM
IndentWidth: 8
UseTab: Always
BreakBeforeBraces: Linux
AllowShortIfStatementsOnASingleLine: false
IndentCaseLabels: false
AlwaysBreakBeforeMultilineStrings: true
AllowShortBlocksOnASingleLine: false

ContinuationIndentWidth: 8

.gitignore

echo-client
echo-server

README.md

SOCK_STREAM latency test for gvisor

You need: gnuplot, python2, docker

Docker needs to have config like this /etc/docker/daemon.json:

{
    "runtimes": {
        "runsc-kvm": {
            "path": "/usr/local/bin/runsc",
            "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds"]
        },
        "runsc-kvm-host": {
            "path": "/usr/local/bin/runsc",
            "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds", "--network=host"]
        },
        "runsc-kvm-vfs2": {
            "path": "/usr/local/bin/runsc",
            "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds", "--vfs2"]
        }
    },
  "ipv6": true,
  "fixed-cidr-v6": "fd80:1::/64"
}

To run:

git clone https://gist.github.com/majek/a2cea2d116fa04aaf167f45f357b4311 echo
make -C echo
./echo/test_perf.sh

common.h

#define MIN(a, b) ((a) < (b) ? (a) : (b))

#define BUFFER_SIZE (128 * 1024)

/* net.c */
int net_parse_sockaddr(struct sockaddr_storage *ss, const char *addr);
int net_connect_blocking(struct sockaddr_storage *sas, int do_zerocopy);
const char *net_ntop(struct sockaddr_storage *ss);
int net_bind(struct sockaddr_storage *ss);
int net_accept(int sd, struct sockaddr_storage *ss);

/* inlines */
#define TIMESPEC_NSEC(ts) ((ts)->tv_sec * 1000000000ULL + (ts)->tv_nsec)
inline static uint64_t realtime_now()
{
	struct timespec now_ts;
	clock_gettime(CLOCK_MONOTONIC, &now_ts);
	return TIMESPEC_NSEC(&now_ts);
}

echo-client.c

#include <errno.h>
#include <error.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>

#include "common.h"

static int trunc_iov(struct iovec *iov, int iov_cnt, int b_sz, int desired)
{
	int i;
	for (i = 0; i < iov_cnt; i++) {
		int m = MIN(desired, b_sz);
		iov[i].iov_len = m;
		desired -= m;
		if (desired == 0) {
			return i + 1;
		}
	}
	return iov_cnt;
}

int main(int argc, char **argv)
{
	if (argc < 2) {
		fprintf(stderr,
			"Usage: %s <target> <amount in KiB> <number of "
			"bursts>\n",
			argv[0]);
		exit(-1);
	}

	struct sockaddr_storage target;
	net_parse_sockaddr(&target, argv[1]);

	uint64_t burst_sz = 1 * 1024 * 1024; // 1MiB
	if (argc > 2) {
		char *endptr;
		double ts = strtod(argv[2], &endptr);
		if (ts < 0 || *endptr != '\0') {
			error(-1, 0, "Can't parse number %s", argv[2]);
		}
		if (ts < 1.0) {
			burst_sz = BUFFER_SIZE;
		} else {
			burst_sz = ts * 1024.0; // In KiB
		}
	}

	long burst_count = 1;
	if (argc > 3) {
		char *endptr;
		burst_count = strtol(argv[3], &endptr, 10);
		if (burst_count < 0 || *endptr != '\0') {
			error(-1, 0, "Can't parse number %s", argv[2]);
		}
	}

	fprintf(stderr, "[+] Sending %ld blocks of %.1fMiB to %s\n",
		burst_count, burst_sz / (1024 * 1024.), net_ntop(&target));

	int fd = net_connect_blocking(&target, 1);
	if (fd < 0) {
		error(-1, errno, "connect()");
	}

	sleep(1);

	int val = 10 * 1000; // 10 ms, in us. requires CAP_NET_ADMIN
	int r = setsockopt(fd, SOL_SOCKET, SO_BUSY_POLL, &val, sizeof(val));
	if (r < 0) {
		if (errno == EPERM) {
			fprintf(stderr,
				"[ ] Failed to set SO_BUSY_POLL. Are you "
				"CAP_NET_ADMIN?\n");
		} else {
			error(-1, errno,
			      "setsockopt(SOL_SOCKET, SO_BUSY_POLL)");
		}
	}

	/* Attempt to set large TX and RX buffer. Why not. */
	val = burst_sz * 2;
	setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &val, sizeof(val));
	setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &val, sizeof(val));

	char tx_buf[BUFFER_SIZE];
	char rx_buf[BUFFER_SIZE];
	memset(tx_buf, 'a', sizeof(tx_buf));

	int iov_cnt = (burst_sz / sizeof(tx_buf)) + 1;
	struct iovec tx_iov[iov_cnt];
	struct iovec rx_iov[iov_cnt];
	int i;
	for (i = 0; i < iov_cnt; i++) {
		tx_iov[i] = (struct iovec){.iov_base = tx_buf,
					   .iov_len = sizeof(tx_buf)};
		rx_iov[i] = (struct iovec){.iov_base = rx_buf,
					   .iov_len = sizeof(rx_buf)};
	}

	uint64_t total_t0 = realtime_now();

	int burst_i;
	for (burst_i = 0; burst_i < burst_count; burst_i += 1) {
		uint64_t t0 = realtime_now();

		uint64_t tx_bytes = burst_sz;
		uint64_t rx_bytes = burst_sz;

		while (tx_bytes || rx_bytes) {
			if (tx_bytes) {
				int d = trunc_iov(
					tx_iov, iov_cnt, sizeof(tx_buf),
					MIN(128 * 1024 * 1024, tx_bytes));
				struct msghdr msg_hdr = {
					.msg_iov = tx_iov,
					.msg_iovlen = d,
				};
				int n = sendmsg(fd, &msg_hdr, MSG_DONTWAIT);
				if (n < 0) {
					if (errno == EINTR) {
						continue;
					}
					if (errno == ECONNRESET) {
						fprintf(stderr,
							"[!] ECONNRESET\n");
						break;
					}
					if (errno == EPIPE) {
						fprintf(stderr, "[!] EPIPE\n");
						break;
					}
					if (errno == EAGAIN) {
						// pass
					} else {
						error(-1, errno, "send()");
					}
				}
				if (n == 0) {
					error(-1, errno, "?");
				}
				if (n > 0) {
					tx_bytes -= n;
				}
			}

			if (rx_bytes) {
				int flags = MSG_DONTWAIT;
				if (tx_bytes == 0) {
					// block. allright. Let busy_poll do
					// work from here.
					// flags = MSG_WAITALL;
					flags = 0;
				}

				int d = trunc_iov(
					rx_iov, iov_cnt, sizeof(rx_buf),
					MIN(128 * 1024 * 1024, rx_bytes));
				struct msghdr msg_hdr = {
					.msg_iov = rx_iov,
					.msg_iovlen = d,
				};
				int n = recvmsg(fd, &msg_hdr, flags);
				if (n < 0) {
					if (errno == EINTR) {
						continue;
					}
					if ((flags & MSG_DONTWAIT) != 0 &&
					    errno == EAGAIN) {
						continue;
					}
					error(-1, errno, "recvmsg()");
				}
				if (n == 0) {
					error(-1, errno, "?");
				}
				if (n > 0) {
					rx_bytes -= n;
				}
			}
		}

		uint64_t t1 = realtime_now();
		printf("%ld\n", (t1 - t0) / 1000);
	}
	close(fd);

	uint64_t total_t1 = realtime_now();

	fprintf(stderr, "[+] Wrote %ld bursts of %.1fMiB in %.1fms\n",
		burst_count, burst_sz / (1024 * 1024.),
		(total_t1 - total_t0) / 1000000.);
	return 0;
}

echo-server.c

#include <errno.h>
#include <error.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>

#include "common.h"

int main(int argc, char **argv)
{
	if (argc < 2) {
		fprintf(stderr, "Usage: %s <listen>\n", argv[0]);
		exit(-1);
	}

	struct sockaddr_storage listen;
	net_parse_sockaddr(&listen, argv[1]);

	int busy_poll = 0;
	if (argc > 2) {
		busy_poll = 1;
	}

	fprintf(stderr, "[+] Accepting on %s busy_poll=%d\n", net_ntop(&listen),
		busy_poll);

	int sd = net_bind(&listen);
	if (sd < 0) {
		error(-1, errno, "connect()");
	}

	int val = BUFFER_SIZE * 2;
	setsockopt(sd, SOL_SOCKET, SO_SNDBUF, &val, sizeof(val));
	setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &val, sizeof(val));

again_accept:;
	struct sockaddr_storage client;
	int cd = net_accept(sd, &client);

	if (busy_poll) {
		int val = 10 * 1000; // 10 ms, in us. requires CAP_NET_ADMIN
		int r = setsockopt(cd, SOL_SOCKET, SO_BUSY_POLL, &val,
				   sizeof(val));
		if (r < 0) {
			if (errno == EPERM) {
				fprintf(stderr,
					"[ ] Failed to set SO_BUSY_POLL. Are "
					"you "
					"CAP_NET_ADMIN?\n");
			} else {
				error(-1, errno,
				      "setsockopt(SOL_SOCKET, SO_BUSY_POLL)");
			}
		}
	}

	uint64_t t0 = realtime_now();

	char buf[BUFFER_SIZE];

	uint64_t sum = 0;
	while (1) {
		int n = recv(cd, buf, sizeof(buf), 0);
		if (n < 0) {
			if (errno == EINTR) {
				continue;
			}
			if (errno == ECONNRESET) {
				fprintf(stderr, "[!] ECONNRESET\n");
				break;
			}
			error(-1, errno, "read()");
		}

		if (n == 0) {
			/* On TCP socket zero means EOF */
			fprintf(stderr, "[-] edge side EOF\n");
			break;
		}

		sum += n;

		int m = send(cd, buf, n, MSG_NOSIGNAL);
		if (m < 0) {
			if (errno == EINTR) {
				continue;
			}
			if (errno == ECONNRESET) {
				fprintf(stderr, "[!] ECONNRESET on origin\n");
				break;
			}
			if (errno == EPIPE) {
				fprintf(stderr, "[!] EPIPE on origin\n");
				break;
			}
			error(-1, errno, "send()");
		}
		if (m == 0) {
			break;
		}
		if (m != n) {
			int err;
			socklen_t err_len = sizeof(err);
			int r = getsockopt(cd, SOL_SOCKET, SO_ERROR, &err,
					   &err_len);
			if (r < 0) {
				error(-1, errno, "getsockopt()");
			}
			errno = err;
			if (errno == EPIPE || errno == ECONNRESET) {
				break;
			}
			error(-1, errno, "send()");
		}
	}

	close(cd);
	uint64_t t1 = realtime_now();

	fprintf(stderr, "[+] Read %.1fMiB in %.1fms\n", sum / (1024 * 1024.),
		(t1 - t0) / 1000000.);
	goto again_accept;

	return 0;
}

Makefile

all:
	clang -g -Wall -Wextra -O3 net.c echo-client.c -o echo-client
	clang -g -Wall -Wextra -O3 net.c echo-server.c -o echo-server

mmhistogram

#!/usr/bin/env python2
import argparse
import itertools
import math
import sys

parser = argparse.ArgumentParser(description='Print log-2 histogram, like systemtap')
parser.add_argument('-t', '--title', default="Values",
                    help='title to print')
parser.add_argument('-c', '--columns', type=int, default="50",
                    help='number of colums for')
parser.add_argument('-b', '--base', type=int, default="2",
                    help='log base')
parser.add_argument('-l', '--linear', action='store_true',
                    help='do linear, not log')
parser.add_argument('-j', '--justval', action='store_true',
                    help='ignore bounds, use values from input')
parser.add_argument('-p', '--percentage', action='store_true',
                    help='Print percentage instead of counts')
parser.add_argument('--csv', action='store_true')
args = parser.parse_args()


M = []
for line in sys.stdin:
    line = line.strip()
    try:
        v = float(line)
    except:
        continue
    M.append( v )

M.sort(reverse=True)

totalcount = len(M)
minval = M[-1]
maxval = M[0]
avgval = sum(M) / float(len(M))
devval = math.sqrt(sum([(m - avgval)**2 for m in M]) / float(len(M)))

medval = M[len(M)/2]

KV = []

for i in itertools.count():
    if args.justval:
        bound=M[-1]
        boundb=M[-1]
    elif not args.linear:
        if i > 0:
            boundb = args.base**(i-1)
        else:
            boundb = 0
        bound = args.base**i
    else:
        boundb = args.base*(i)
        bound = args.base*(i+1)

    c = 0
    while len(M) > 0 and bound >= M[-1] :
        c += 1
        M.pop()
    KV.append( (boundb, bound, c) )

    if not M:
        break

maxcount = float(max(c for _, _, c in KV))
maxbound = KV[-1][0]
boundl = max(len(str(maxbound)), len('value'))

if args.csv == False:
    print "%s min:%.2f avg:%.2f med=%.2f max:%.2f dev:%.2f count:%d" % (
        args.title,
        minval,
        avgval,
        medval,
        maxval,
        devval,
        totalcount,
    )
    print "%s:" % (
        args.title,
    )
    print "%*s |%*s %s" % (
        boundl+1,
        "value",
        args.columns,
        "-" * args.columns,
        "count"
    )

    for boundb, bound, c in KV:
        if args.percentage:
            cp = "%5.2f%%" % ((c / float(totalcount))*100.0,)
        else:
            cp = "%d" % (c,)
        print "%*d |%*s %s" % (
            boundl + 1,
            boundb,
            args.columns,
            "*" * int(args.columns * (c/maxcount)),
            cp
        )
else:
    print "%s, %.2f, %.2f, %.2f, %.2f, %.2f, %d" % (
        args.title,
        minval,
        avgval,
        medval,
        maxval,
        devval,
        totalcount,
    )

net.c

#include <arpa/inet.h>
#include <errno.h>
#include <error.h>
#include <fcntl.h>
#include <netinet/tcp.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/un.h>
#include <time.h>

//#include "common.h"

static int net_addr_from_name_inet(struct sockaddr_storage *ss,
				   const char *host)
{
	struct sockaddr_in *sin = (struct sockaddr_in *)ss;
	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;

	if (inet_pton(AF_INET, host, &sin->sin_addr) == 1) {
		sin->sin_family = AF_INET;
		return 0;
	}

	if (inet_pton(AF_INET6, host, &sin6->sin6_addr) == 1) {
		sin6->sin6_family = AF_INET6;
		return 0;
	}
	return -1;
}

static int net_addr_from_name_un(struct sockaddr_storage *ss, const char *host)
{
	// AF_UNIX
	struct sockaddr_un *sun = (struct sockaddr_un *)ss;
	sun->sun_family = AF_UNIX;
	strncpy(sun->sun_path, host, sizeof(sun->sun_path));
	if (sun->sun_path[0] == '@') {
		// Linux abstract sockets often use @ for zero
		sun->sun_path[0] = '\x00';
	}
	return 0;
}

int net_parse_sockaddr(struct sockaddr_storage *ss, const char *addr)
{
	memset(ss, 0, sizeof(struct sockaddr_storage));

	char *colon = strrchr(addr, ':');
	if (colon == NULL || colon[1] == '\0') {
		goto af_unix;
	}

	char *endptr;
	long port = strtol(&colon[1], &endptr, 10);
	if (port < 0 || port > 65535 || *endptr != '\0') {
		goto af_unix;
	}

	char tmp[255];
	char *host = tmp;
	int addr_len = colon - addr > 254 ? 254 : colon - addr;
	strncpy(host, addr, addr_len);
	host[addr_len] = '\0';
	if (host[0] == '[' && addr_len > 1 && host[addr_len - 1] == ']') {
		host[addr_len - 1] = '\0';
		host += 1;
	}
	int r = net_addr_from_name_inet(ss, host);
	if (r != 0) {
	af_unix:
		net_addr_from_name_un(ss, addr);
	}

	struct sockaddr_in *sin = (struct sockaddr_in *)ss;
	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;

	switch (ss->ss_family) {
	case AF_INET:
		sin->sin_port = htons(port);
		break;
	case AF_INET6:
		sin6->sin6_port = htons(port);
		break;
	}
	return -1;
}

#define SOCKADDR_UN_SIZE(sun)                                                  \
	((sun)->sun_path[0] == '\x00'                                          \
		 ? __builtin_offsetof(struct sockaddr_un, sun_path) + 1 +      \
			   strnlen(&(sun)->sun_path[1],                        \
				   sizeof((sun)->sun_path))                    \
		 : sizeof(struct sockaddr_un))

static size_t sizeof_ss(struct sockaddr_storage *ss)
{
	switch (ss->ss_family) {
	case AF_INET:
	case AF_INET6:
		return sizeof(struct sockaddr_storage);
	case AF_UNIX:
		return SOCKADDR_UN_SIZE((struct sockaddr_un *)ss);
	default:
		abort();
	}
}

int net_connect_blocking(struct sockaddr_storage *ss, int do_zerocopy)
{
	int sd = socket(ss->ss_family, SOCK_STREAM, 0);
	if (sd < 0) {
		error(-1, errno, "socket()");
	}

	if (ss->ss_family == AF_INET || ss->ss_family == AF_INET6) {
		/* Don't buffer partial packets */
		int one = 1;
		int r = setsockopt(sd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
		if (r < 0) {
			error(-1, errno, "setsockopt()");
		}

		/* Cubic is a bit more stable in tests than bbr */
		char *cong = "cubic";
		r = setsockopt(sd, SOL_TCP, TCP_CONGESTION, cong, strlen(cong));
		if (r < 0) {
			error(-1, errno, "setsockopt(TCP_CONGESTION)");
		}

		if (do_zerocopy) {
			/* Zerocopy shall be set on the parent accept socket. */
			int one = 1;
			int r = setsockopt(sd, SOL_SOCKET, SO_ZEROCOPY, &one,
					   sizeof(one));
			if (r < 0) {
				error(-1, errno, "getsockopt(SO_ZEROCOPY)");
			}
		}
	}

again:;
	int r = connect(sd, (struct sockaddr *)ss, sizeof_ss(ss));
	if (r < 0) {
		if (errno == EINTR) {
			goto again;
		}
		error(-1, errno, "connect()");
	}

	return sd;
}

int net_getpeername(int sd, struct sockaddr_storage *ss)
{
	memset(ss, 0, sizeof(struct sockaddr_storage));

	socklen_t ss_len = sizeof(struct sockaddr_storage);
	int r = getpeername(sd, (struct sockaddr *)ss, &ss_len);
	if (r < 0) {
		error(-1, errno, "getpeername()");
	}

	/* stick trailing zero to AF_UNIX name */
	if (ss_len < sizeof(struct sockaddr_storage)) {
		((char *)ss)[ss_len] = '\0';
	}

	return 0;
}

int net_getsockname(int sd, struct sockaddr_storage *ss)
{
	memset(ss, 0, sizeof(struct sockaddr_storage));

	socklen_t ss_len = sizeof(struct sockaddr_storage);
	int r = getsockname(sd, (struct sockaddr *)ss, &ss_len);
	if (r < 0) {
		error(-1, errno, "getsockname()");
	}

	/* stick trailing zero to AF_UNIX name */
	if (ss_len < sizeof(struct sockaddr_storage)) {
		((char *)ss)[ss_len] = '\0';
	}

	return 0;
}

const char *net_ntop(struct sockaddr_storage *ss)
{
	char s[sizeof(struct sockaddr_storage) + 1];
	static char a[sizeof(struct sockaddr_storage) + 32];
	struct sockaddr_in *sin = (struct sockaddr_in *)ss;
	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
	struct sockaddr_un *sun = (struct sockaddr_un *)ss;
	int port;
	const char *r;
	switch (ss->ss_family) {
	case AF_INET:
		port = htons(sin->sin_port);
		r = inet_ntop(sin->sin_family, &sin->sin_addr, s, sizeof(s));
		if (r == NULL) {
			error(-1, errno, "inet_ntop()");
		}
		snprintf(a, sizeof(a), "inet://%s:%i", s, port);
		break;
	case AF_INET6:
		r = inet_ntop(sin6->sin6_family, &sin6->sin6_addr, s,
			      sizeof(s));
		if (r == NULL) {
			error(-1, errno, "inet_ntop()");
		}
		port = htons(sin6->sin6_port);
		snprintf(a, sizeof(a), "inet6://[%s]:%i", s, port);
		break;
	case AF_UNIX:
		memcpy(s, sun->sun_path, sizeof(sun->sun_path));
		s[sizeof(sun->sun_path)] = '\x00';
		if (s[0] == '\x00') {
			s[0] = '@';
		}
		snprintf(a, sizeof(a), "unix://%s", s);
		break;
	default:
		abort();
	}
	return a;
}

int net_bind(struct sockaddr_storage *ss)
{
	int sd = socket(ss->ss_family, SOCK_STREAM, 0);
	if (sd < 0) {
		error(-1, errno, "socket()");
	}

	int one = 1;
	int r = setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (char *)&one,
			   sizeof(one));
	if (r < 0) {
		error(-1, errno, "setsockopt(SO_REUSEADDR)");
	}

	if (ss->ss_family == AF_INET || ss->ss_family == AF_INET6) {
		one = 1;
		r = setsockopt(sd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
		if (r < 0) {
			error(-1, errno, "setsockopt()");
		}

		/* Cubic is a bit more stable in tests than bbr */
		char *cong = "cubic";
		r = setsockopt(sd, SOL_TCP, TCP_CONGESTION, cong, strlen(cong));
		if (r < 0) {
			error(-1, errno, "setsockopt(TCP_CONGESTION)");
		}
	}

	if (ss->ss_family == AF_INET6) {
		one = 1;
		r = setsockopt(sd, IPPROTO_IPV6, IPV6_V6ONLY, &one,
			       sizeof(one));
		if (r < 0) {
			error(-1, errno, "setsockopt(IPV6_V6ONLY)");
		}
	}

	r = bind(sd, (struct sockaddr *)ss, sizeof_ss(ss));
	if (r < 0) {
		error(-1, errno, "bind()");
	}

	listen(sd, 1024);
	return sd;
}

int net_accept(int sd, struct sockaddr_storage *ss)
{
again_accept:;
	socklen_t ss_len = sizeof(struct sockaddr_storage);
	int cd = accept(sd, (struct sockaddr *)ss, &ss_len);
	if (cd < 0) {
		if (errno == EINTR) {
			goto again_accept;
		}
		error(-1, errno, "accept()");
	}

	/* stick trailing zero to AF_UNIX name */
	if (ss_len < sizeof(struct sockaddr_storage)) {
		((char *)ss)[ss_len] = '\0';
	}
	return cd;
}

void set_nonblocking(int fd)
{
	int flags, ret;
	flags = fcntl(fd, F_GETFL, 0);
	if (-1 == flags) {
		flags = 0;
	}
	ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
	if (-1 == ret) {
		error(-1, errno, "fcntl(O_NONBLOCK)");
	}
}

test_perf.sh

#!/bin/bash
# requires /etc/docker/daemon.json like
# {
#     "runtimes": {
#         "runsc-kvm": {
#             "path": "/usr/local/bin/runsc",
#             "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds"]
#         },
#         "runsc-kvm-host": {
#             "path": "/usr/local/bin/runsc",
#             "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds", "--network=host"]
#         },
#         "runsc-kvm-vfs2": {
#             "path": "/usr/local/bin/runsc",
#             "runtimeArgs": ["--platform=kvm", "--fsgofer-host-uds", "--vfs2"]
#         }
#     },
#   "ipv6": true,
#   "fixed-cidr-v6": "fd80:1::/64"
# }


PORT=1231

./echo/echo-server 0.0.0.0:$PORT 2>/dev/null &
A=$!
./echo/echo-server [::]:$PORT 2>/dev/null &
B=$!
rm echo/pipe
./echo/echo-server ./echo/pipe 2>/dev/null &
C=$!

function cleanup()
{
    kill $A
    kill $B
    kill $C
}

trap cleanup EXIT

# Size of block to send over the SOCK_STREAM connection
SZ=4096

# Number of blocks to send
NR=40


sleep 1
IP4=`ip route get 1 | awk '{print $(7);exit}'`
IP6=`ip -6 r get 2606:4700:4700::1001|awk '{print $(11);exit}'`


DOCKER_KVM="docker run --runtime=runsc-kvm --rm --volume $PWD/echo:/echo -it ubuntu:bionic"
DOCKER_KVM_HOST="docker run --runtime=runsc-kvm-host --rm --volume $PWD/echo:/echo -it ubuntu:bionic"
DOCKER_KVM_VFS2="docker run --runtime=runsc-kvm-vfs2 --rm --volume $PWD/echo:/echo -it ubuntu:bionic"

function do_run()
{
    $DOCKER_KVM /echo/echo-client '/echo/pipe' $SZ $NR \
        | ./echo/mmhistogram --csv -t "uds"

    $DOCKER_KVM_HOST /echo/echo-client $IP6:$PORT $SZ $NR \
        | ./echo/mmhistogram --csv -t "v6 net=host"

    $DOCKER_KVM_HOST /echo/echo-client $IP4:$PORT $SZ $NR \
        | ./echo/mmhistogram --csv -t "v4 net=host"

    $DOCKER_KVM /echo/echo-client $IP6:$PORT $SZ $NR \
        | ./echo/mmhistogram --csv -t "v6 net=netstack"

    $DOCKER_KVM /echo/echo-client $IP4:$PORT $SZ $NR \
        | ./echo/mmhistogram --csv -t "v4 net=netstack"

    $DOCKER_KVM bash -c "(/echo/echo-server '/tmp/pipe' > /dev/null & ); sleep 1; /echo/echo-client '/tmp/pipe' $SZ $NR" \
        | ./echo/mmhistogram --csv -t "uds"

    $DOCKER_KVM_VFS2 bash -c "(/echo/echo-server '/tmp/pipe' > /dev/null & ); sleep 1; /echo/echo-client '/tmp/pipe' $SZ $NR" \
        | ./echo/mmhistogram --csv -t "uds vfs2"

    $DOCKER_KVM_HOST bash -c "(/echo/echo-server '[::1]:1234' > /dev/null & ); sleep 1; /echo/echo-client '[::1]:1234' $SZ $NR" \
        | ./echo/mmhistogram --csv -t "v6 net=host"

    $DOCKER_KVM_HOST bash -c "(/echo/echo-server 127.0.0.1:1234 > /dev/null & ); sleep 1; /echo/echo-client 127.0.0.1:1234 $SZ $NR" \
        | ./echo/mmhistogram --csv -t "v4 net=host"

    $DOCKER_KVM bash -c "(/echo/echo-server '[::1]:1234' > /dev/null & ); sleep 1; /echo/echo-client '[::1]:1234' $SZ $NR" \
        | ./echo/mmhistogram --csv -t "v6 net=netstack"

    $DOCKER_KVM bash -c "(/echo/echo-server 127.0.0.1:1234 > /dev/null & ); sleep 1; /echo/echo-client 127.0.0.1:1234 $SZ $NR" \
        | ./echo/mmhistogram --csv -t "v4 net=netstack"
}

function echo_plot() {
    cat <<EOF
set xrange [-0.5:15+0.5];
set yrange [0:];
set title sprintf("SOCK\_STREAM echo latency for %s KiB block, repeated %s times, platform kvm", ARG1, ARG2)
set terminal pngcairo transparent enhanced background rgb 'white' linewidth 2 font 'arial,15' size 1000, 600
set xtics nomirror scale 0 offset -1  rotate by -45
set grid ytics
set ylabel "Latency in us"
set datafile separator ","
set output ARG4
set bars 8.0
set format y "%.1s*10^{%T}"

FIRST=4
plot \
	ARG3 every ::0::FIRST using (\$0+0.5):(\$3-\$6/2):(\$5):(\$2):(\$3+\$6/2):xtic(1) with candlesticks title "server in host", \
	ARG3 every ::(FIRST+1) using (FIRST+1+\$0+0.5):(\$3-\$6/2):(\$5):(\$2):(\$3+\$6/2):xtic(1) with candlesticks title "server in guest"

EOF
}

DATA=$(mktemp --suffix=.csv)
do_run | tee $DATA

gnuplot -c <(echo_plot) $SZ $NR $DATA numbers-$NR-$SZ.png \
    && xdg-open numbers-$NR-$SZ.png

rm $DATA