TCP non-blocking client connection using kqueue

CreateTCPSocket.c

#include "tcpsocket.h"
/**
 * @brief Create a Socket object
 * @name CreateSocket
 * @param int domain 
 * @param bool keepalive 
 * @param bool nonblock 
 * @return int socket
 */
int CreateSocket(int domain, bool keepalive, bool nonblock) {
    int optval;
    socklen_t optlen = sizeof(optval);
    static int s, on = 1;
    if ((s = socket(domain, SOCK_STREAM, 0)) == -1) {
        fprintf(stderr, "creating socket: %s", strerror(errno));
        return NET_ERROR;
    }
    // for server sockets it is usefull to set NONBLOCK flag to true
    if (nonblock) {
        int flags;
	// or we can use NonBlock(int fd) method...
        if ((flags = fcntl(s, F_GETFL)) == -1) {
            fprintf(stderr, "fcntl(F_GETFL): %s", strerror(errno));
            return NET_ERROR;
        }
        if (fcntl(s, F_SETFL, flags | O_NONBLOCK) == -1) {
            fprintf(stderr, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno));
            return NET_ERROR;
        }   
    }
    /* Make sure connection-intensive things like the redis benckmark
     * will be able to close/open sockets a zillion of times */
    if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
        fprintf(stderr, "setsockopt SO_REUSEADDR: %s", strerror(errno));
        return NET_ERROR;
    }
    if (keepalive) {
        if (getsockopt(s, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen) < 0) {
            fprintf(stderr, "getsockopt SO_KEEPALIVE failed: %s", strerror(errno));
            close(s);
            return NET_ERROR;
        }

        optval = 1;
        optlen = sizeof(optval);
        if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen) < 0) {
            fprintf(stderr, "setsockopt SO_KEEPALIVE failed: %s", strerror(errno));
            close(s);
            return NET_ERROR;
        }
    }
    return s;
}
/**
 * @name CreateSocketNB
 * @brief creates non-blocking TCPsocket for client or server.
 * 
 * @param int domain - domain argument specifies a communications domain this selects the protocol family which
 *                     should be used.
 * @param bool isServer - boolean flag determines that the socket is going to be used by TCP server or not.
 * @return int socket if all went OK, otherwise NET_ERROR (-1) will be sent back to the caller
 */
int CreateSocketNB(int domain, bool isServer) {
    int optval;
    socklen_t optlen = sizeof(optval);
    static int s, on = 1;
    if ((s = socket(domain, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) {
        fprintf(stderr, "creating socket: %s", strerror(errno));
        return NET_ERROR;
    }
    if (isServer) {
        /* 
         * Make sure connection-intensive things like the redis benckmark
         * will be able to close/open sockets a zillion of times 
         */
        if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
            fprintf(stderr, "setsockopt SO_REUSEADDR: %s", strerror(errno));
            return NET_ERROR;
        }
    }
    optval = 1;
    optlen = sizeof(optval);
    if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen) < 0) {
        fprintf(stderr, "setsockopt SO_KEEPALIVE failed: %s", strerror(errno));
        close(s);
        return NET_ERROR;
    }
    int val = 1;
    assert(0 == setsockopt(s, 0, TCP_NODELAY, (char*)&optval, optlen));
    return s;
}

myevent.c

#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <sys/event.h>
/**
 * @brief Registers socket to kqueue write event handler
 * @name Register_write_Socket
 * @param int _kq - kqueue descriptor
 * @param struct kevent *_change_event - change list
 * @param int socket - the socket to be registered
 * @param void *ctx - event handler context
 */
void Register_write_Socket(int _kq, struct kevent* _change_event, int socket, void* ctx) {
    EV_SET(_change_event, socket, EVFILT_WRITE, EV_ADD | EV_ENABLE | EV_ERROR | EV_EOF | EV_CLEAR, 0, 0, ctx);
    if (kevent(_kq, _change_event, 1, NULL, 0, NULL) == -1) {
        perror("kevent");
        close(_kq);
        exit(1);
    }
}
/**
 * @brief Deregisters - removes - previously registered write event handler of a socket
 * @name Remove_write_Socket
 * @param int _kq - kqueue descriptor
 * @param struct kevent *_change_event - change list
 * @param int socket - the socket to be removed from event queue handler's list
 */
void Remove_write_Socket(int _kq, struct kevent* _change_event, int socket) {
    EV_SET(_change_event, socket, EVFILT_WRITE, EV_DELETE, 0, 0, 0);
    if (kevent(_kq, _change_event, 1, NULL, 0, NULL) == -1) {
        perror("kevent");
        close(_kq);
        exit(1);
    }
}

myevent.h

#ifndef __MYEVENT_H__
#define __MYEVENT_H__
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <sys/event.h>
void Register_write_Socket(int _kq, struct kevent* _change_event, int socket, void* ctx);
void Remove_write_Socket(int _kq, struct kevent* _change_event, int socket);
#endif //__MYEVENT_H__

TCPClientConnect.c

#include "tcpsocket.h"
#include "myevent.h"
#include <stdbool.h>
/**
 * @brief Set socket non-blocking
 * @name NonBlock
 * @param int fd - the socket
 * @return int tretval 0 if success, -1 if does not
 */
int NonBlock(int fd) {
    int flags;
    if ((flags = fcntl(fd, F_GETFL)) == -1) {
        fprintf(stderr, "fcntl(F_GETFL): %s", strerror(errno));
        return NET_ERROR;
    }
    if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) {
        fprintf(stderr, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno));
        return NET_ERROR;
    }
    return NET_OK;
}
/**
 * @name TCPClientConnect
 * @brief this method tries to establish TCP connection to the remote server - it uses non-blocking socket from 
 *        the result of CreateSocket function. To be able to determine that everything went good during the connection
 *        request, this function utilizes kqueue event handler to do that. More on this in the code below.
 * @param char* address - address string or FQDN host name of the remote site.
 * @param int port - the listening port of the remote site.
 * @return int connected_socket if success, otherwise NET_ERROR (-1)
 */
int TCPClientConnect(char *address, int port) {
    int s, _kq;					// local kqueue handler
    struct kevent _event[1], _events[1];	// for kqueue
    struct sockaddr_in sa;			// socket address structure
    // We create the non-blocking socket first
    if ((s = CreateSocket(AF_INET,true,false)) == NET_ERROR) {
        return NET_ERROR;
    }
    // Specify the address family
    sa.sin_family = AF_INET;			// domain AF_INET
    // add port number
    sa.sin_port = htons(port);			// register port number
    // try to resolve IP address string, if it fails, then resolve by name
    if (inet_aton(address, &sa.sin_addr) == 0) {	// if the address conforms "aaa.bbb.ccc.ddd", it should be 0
        struct hostent *he;			// otherwise we try to resolve as hostname - string like FQDN
        he = gethostbyname(address);		// call help from system's resolver
        if (he == NULL) {			// it could not resolve the hostname - it is not in the known hosts list
            fprintf(stderr, "can't resolve: %s", address);
            close(s);				// close socket
            return NET_ERROR;			// return -1
        }
        memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr));
    }
    // we got the remote address info, try to connect
    if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
        fprintf(stdout, " - Connection message: [%d] %s", errno, strerror(errno));
        if (errno == 0) {	// if no error, we successfully connected
	    NonBlock(s);	// set our socket non-blocking
            return s;		// return socket
        } else if (errno == EINPROGRESS) {
	    // Because the socket is set to NON-BLOCKING, there is a chance to get some error during the attempt...
	    // In that case when we want to run this code on Linux, then we need to call the following function
	    // to get the process status: getsockopt(..., SOL_SOCKET, SO_ERROR, ...)
	    // otherwise we could miss the reason of the non blocking socket connection failure - connect returns
	    // -1 if the request is still pending due to some reasons. So, we definitely need to wait the response
	    // from the OS about the last connection request. If any other kind of error comes back at the end of
	    // connecting, than we need to sign that this connection request failed, and probably we need to try it
	    // again later on. Maybe we can setup a timer, and a connection status check count - which is decrementing
	    // every getsockopt call - (if we do not use kqueue), and if it reaches zero, then we can conclude that the 
	    // connection to the remote size is not OK for this time.
	    // If we use kqueue however we can skip that test by registering new socket to the write event queue list.
	    // What does it mean? Well, if we register our non-blocking socket to the write list we can wait - either 
	    // specifying some valuable timeout - or let it wait indefinitely until something happens to the connecting socket.
	    // Latter case it will - most probably - refused, or rejected, beacuse event.fflags contains the real
	    // reason of the error (as it would set by connect by default) which is not 0 definitely. 
	    // So, until it is 0, then we continue to wait the positive or negative response from OS about the 
	    // pending connect status.
            Kqueue((int *)&_kq);	// utilize kqueue
            assert(-1 != _kq);		// check it is correctly created
	    // we register the writer event on the socket to see if error occurs or not during the connecting process
            Register_write_Socket(_kq, &_event[0], s, NULL);	// register new socket  in kqueue's write event handler
            struct kevent events[1];
	    // Wait for the event on the write end, if something goes wrong, the event handler will detect...
	    // bool quit = false;
	    // for (;;) {
	    //     if (quit) {
	    //         break;
	    //     }
	        int n = kevent(_kq, NULL, 0, _events, 1, NULL);
                if (_events[0].filter == EVFILT_WRITE) {
                    errno = 0;
		    if (_events[0].flags & EV_EOF) {
                        errno = _events[0].fflags;
	    //	    if (errno == 0) {
	    //          continue;	// it is still in progress, wait if flags becomes 0 or not EINPROGRESS
	    //      }
                        if (errno != EINPROGRESS) {
			    // quit = true;
                            fprintf(stderr, " - connect: %s", strerror(errno));
                            close(s);
                            close(_kq);
                            return NET_ERROR;            
                        }
                    }
                }
	    // } // end of infinite for loop
        } else {
            debug_error(" - connect: %s", strerror(errno));
            close(s);
            close(_kq);
            return NET_ERROR;
        }
    }
    close(_kq);
    return s;
}

tcpsocket.h

#ifndef _WRAPSOCK_H
#define _WRAPSOCK_H
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/select.h>
#include <sys/event.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netdb.h>
#include <fcntl.h>
#include <netinet/tcp.h>
#include <sys/un.h>
#include <sys/stat.h>

#ifndef NET_ERROR
#define NET_ERROR  -1
#endif // !NET_ERROR

#ifndef NET_OK
#define NET_OK      0
#endif // !NET_OK

int NonBlock(int fd);
int TCPClientConnect(char *address, int port);
int CreateSocket(int domain, bool keepalive, bool nonblock);
int CreateSocketNB(int domain, bool isServer);

#endif /* _WRAPSOCK_H */

Connection issues with non-blocking TCP client socket coped with kqueue

This is how I solved the non-blocking TCP client connection difficulties - got and misshandled errors during the connections.

Now extended with all needed wrappers and headers.