epoll client, testing with sockopt SO_RCVLOWAT

epoll_rcvlowat_test_client.c

/*

A few years ago, I tried to create an HTTP client using epoll that could manage many connections. Performance was incredibly poor due to what I determined to be too many syscalls from epoll returning on every packet received. I then moved to FreeBSD and kqueue() and got the performance I was looking for.

I mentioned this on a Hackernews thread announcing a new epoll feature: https://news.ycombinator.com/item?id=17851855#17856351

@caf on HN (@keaston on GitHub) responded saying this was possible using SO_RCVLOWAT. I recall testing this a few years ago, but still could not get it working. Given that it was pretty early in my C days, it's possible I could have been doing a number of things wrong.

@keaston wrote a quick server script to demonstrate this ability. Curious if this also applied to client sockets, I dug up and modified some old code to test it.

Turns out, it works. And I owe Kevin a large number of drinks of his choice.

NOTE: this does error out on most requests/runs. I could debug it, but I'm not going to right now. This is enough to show that SO_RCVLOWAT does, in fact, work when using a socket as a client.

*/


#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <errno.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <stdbool.h>
#include <string.h>


#define HOST "www.google.com"
#define HOST_IP "216.58.217.46"

#define RCVLOWAT_BYTES (16 * 1024)

#define EPOLL_MAXEVENTS 16
#define EPOLL_FLAGS_READ (EPOLLIN|EPOLLET|\
                          EPOLLERR|EPOLLHUP|EPOLLRDHUP)
#define EPOLL_FLAGS_WRITE (EPOLLOUT|EPOLLET|\
                           EPOLLERR|EPOLLHUP|EPOLLRDHUP)


#define DBG_FFL do{printf("\nLINE: %d\n",__LINE__);fflush(stdout);}while(0);
#define DIE(a) do { perror(a); exit(1); } while (0)


//------------------------------------------------------------------------------
int
main()
{
	int ret;
	int sfd;
	struct sockaddr_in sa;
	char buf[RCVLOWAT_BYTES];
	int rcvlowat = RCVLOWAT_BYTES;

	//----------------------------------------------------------
	int efd = epoll_create1(0);
	if (efd == -1)
		DIE("epoll_create");

	struct epoll_event ev;

	//----------------------------------------------------------
	sfd = socket(AF_INET, SOCK_STREAM, 0);
	if (-1 == sfd)
		DIE("socket");

	if (setsockopt(sfd, SOL_SOCKET, SO_RCVLOWAT, &rcvlowat, sizeof rcvlowat))
		DIE("setsockopt");

	// non-blocking client socket
	int flags = fcntl(sfd, F_GETFL, 0);
	if (flags < 0) exit(12);
	fcntl(sfd, F_SETFL, flags | O_NONBLOCK);

	//----------------------------------------------------------
	memset(&sa, '\0', sizeof(sa));
	sa.sin_family = AF_INET;
	sa.sin_addr.s_addr = inet_addr(HOST_IP); // Server IP
	sa.sin_port = htons(80); // Server Port number

	//----------------------------------------------------------
	ret = connect(sfd, (struct sockaddr*) &sa, sizeof(sa));
	if (ret < 0 && errno != EINPROGRESS)
		DIE("connect != EINPROGRESS");

	ev.data.fd = sfd;
	ev.events = EPOLL_FLAGS_WRITE;

	printf("Connected to server %s, port %u\n",
		inet_ntoa(sa.sin_addr), ntohs(sa.sin_port));
	fflush(stdout);

	int s = epoll_ctl(efd, EPOLL_CTL_ADD, sfd, &ev);
	if (s == -1)
		DIE("epoll_ctl");

	// Buffer where events are returned
	struct epoll_event* events = calloc(EPOLL_MAXEVENTS, sizeof ev);


	//----------------------------------------------------------
	// The event loop
	while (1)
	{
		int n = epoll_wait(efd, events, EPOLL_MAXEVENTS, -1);
		if (n < 0 && n == EINTR)
		{
			printf("epoll_wait System call interrupted. Continue..");
			fflush(stdout);
			continue;
		}
		//------------------------------------------------------
		for (int i = 0; i < n; i++)
		{
			//------------------------------------------------------------------
			if ((  events[i].events & EPOLLERR) ||
				(  events[i].events & EPOLLHUP) ||
				(!(events[i].events & (EPOLLIN | EPOLLOUT))))
			{
				// todo: check the error...
				DIE("epoll error");
			}
			//------------------------------------------------------------------
			// hangup
			else if (events[i].events & (EPOLLHUP))
			{
				fprintf(stderr, "DONE: EPOLLHUP()\n");
				fflush(stdout);
			}
			//------------------------------------------------------------------
			// read
			else if (events[i].events & (EPOLLIN))
			{
				do
				{
					ret = read(sfd, buf, sizeof(buf) - 1);
					if ((ret)==-1)
						DIE("err: read\n");

					buf[ret] = '\0';
					printf("\n\nClient Received %d chars - '%s'\n", ret, buf);
					fflush(stdout);
				} while (ret > 0);
			}

			//------------------------------------------------------------------
			//write
			else if (events[i].events & EPOLLOUT)
			{
				const char *request = "GET / HTTP/1.1\n"
				                      "Host: "HOST"\n"
				                      "User Agent: Mozilla/5.0 "
				                      "(compatible; MSIE 9.0; "
				                      "Windows NT 6.1; Trident/5.0)\n"
				                      "Connection: Close\n\n";
				ret = write(sfd, request, strlen(request));
				if ((ret)==-1)
					DIE("write");

				memset(&ev, 0, sizeof(ev));

				ev.events = EPOLL_FLAGS_READ;
				ev.data.fd = events[i].data.fd;
				if (epoll_ctl(efd, EPOLL_CTL_MOD, events[i].data.fd, &ev) < 0)
				{
					printf("ERR: epoll_ctl\nerrno: %d\n", errno);
					fflush(stdout);
					exit(3);
				}
			}
		}
	}

	printf("\n\ndone.\n\n");

	//--------------------------------------------------------------------------
	free(events);
	close(sfd);
	close(efd);
	return 0;
}