Reference implementation for AcceptEx based Event based Overlapped (Asynchronous) I/O TCP Echo Server.

win32_sock.c

#define Win32OutputWSAErrorCode printf
#define Win32OutputErrorCode printf

static int
Win32InitializeWinsock()
{
    WSADATA WSAData;
    if (WSAStartup(MAKEWORD(2, 2), &WSAData) != 0)
    {
        Win32OutputWSAErrorCode("WSAStartup Failed");
        WSACleanup();
        return -1;
    }

    return 0;
}

static win32_mnet_state
Win32InitializeNetworking()
{
    win32_mnet_state Result = {0};
    Result.Initialized = 0;
    Result.NumberOfEvents = 0;

    struct addrinfo Hints;
    ZeroMemory(&Hints, sizeof(Hints));
    Hints.ai_family = AF_INET;
    Hints.ai_socktype = SOCK_STREAM;
    Hints.ai_protocol = IPPROTO_TCP;
    Hints.ai_flags = AI_PASSIVE;

    if (getaddrinfo(NULL, MPROF_SOCK_DEFAULT_PORT, &Hints, &Result.AddressInfo) != 0)
    {
        Win32OutputWSAErrorCode("getaddrinfo Failed");
        WSACleanup();
        return Result;
    }

    // NOTE(Oskar): Initialize socket object
    Result.ListenSocket = WSASocketW(Result.AddressInfo->ai_family, 
                                    Result.AddressInfo->ai_socktype, 
                                    Result.AddressInfo->ai_protocol,
                                    NULL, 0, WSA_FLAG_OVERLAPPED);
    if (Result.ListenSocket == INVALID_SOCKET)
    {
        Win32OutputWSAErrorCode("WSASocket Failed, unable to create listening socket.");
        WSACleanup();
        return Result;
    }

    // NOTE(Oskar): Bind socket to a network address.
    if (bind(Result.ListenSocket, Result.AddressInfo->ai_addr, (int)Result.AddressInfo->ai_addrlen) == SOCKET_ERROR)
    {
        Win32OutputWSAErrorCode("Failed to bind listening socket.");
        WSACleanup();
        return Result;
    }

    // NOTE(Oskar): Listen for incoming requests.
    if (listen(Result.ListenSocket, MPROF_SOCK_DEFAULT_BACKLOG) == SOCKET_ERROR)
    {
        Win32OutputWSAErrorCode("Failed to initiate listening on socket.");
        WSACleanup();
        return Result;
    }

    // NOTE(Oskar): Load the AcceptEx extension function
    GUID AcceptExGuid = WSAID_ACCEPTEX;
    DWORD Bytes = 0;
    int LoadStatus = WSAIoctl(Result.ListenSocket, SIO_GET_EXTENSION_FUNCTION_POINTER,
                              &AcceptExGuid, sizeof(AcceptExGuid),
                              &Result.AcceptEx, sizeof(Result.AcceptEx),
                              &Bytes, NULL, NULL);
    if (LoadStatus == SOCKET_ERROR)
    {
        Win32OutputWSAErrorCode("Failed to load AcceptEx.");
        WSACleanup();
    }

    Result.Initialized = 1;
    return Result;
}

static void
Win32CleanupNetworking(win32_mnet_state *State)
{
    if (State->ListenSocket != INVALID_SOCKET)
    {
        closesocket(State->ListenSocket);
    }

    if (State->LastAcceptSocket != INVALID_SOCKET)
    {
        closesocket(State->LastAcceptSocket);
    }

    if (State->AddressInfo != NULL)
    {
        freeaddrinfo(State->AddressInfo);
    }

    WSACleanup();
}

static void
_Win32SocketConnectionInitialize(win32_mnet_connection *Target, SOCKET LastAcceptSocket)
{
    Target->Socket = LastAcceptSocket;
    ZeroMemory(&(Target->Overlapped), sizeof(OVERLAPPED));
    Target->Operation = MPROF_SOCK_QUEUED_OP_NONE;
    Target->BytesSent = 0;
    Target->BytesReceived = 0;
    Target->DataBuf.len = MPROF_SOCK_BUFFER_SIZE;
    Target->DataBuf.buf = Target->Buffer;
}

// TODO(Oskar): Right now this is Echo specific logic. 
// if doing something interesting with this then this should be changed to actually be event based.
// Basically process the Read/Send/Accept request then go into idle untill code tells it what to do.
static void
Win32NetworkUpdateConnectionState(win32_mnet_connection *SocketConnection, DWORD BytesTransferred)
{
    if (SocketConnection->Operation == MPROF_SOCK_QUEUED_OP_READ)
    {
        SocketConnection->BytesReceived = BytesTransferred;
        SocketConnection->BytesSent = 0;

        SocketConnection->Operation = MPROF_SOCK_QUEUED_OP_SEND;
    }
    else if (SocketConnection->Operation == MPROF_SOCK_QUEUED_OP_SEND)
    {
        SocketConnection->BytesSent += BytesTransferred;
        
        if (SocketConnection->BytesReceived <= SocketConnection->BytesSent)
        {
            // NOTE(Oskar): Done sending
            SocketConnection->Operation = MPROF_SOCK_QUEUED_OP_NONE;
        }
    }

    if (SocketConnection->Operation == MPROF_SOCK_QUEUED_OP_NONE)
    {
        SocketConnection->Operation = MPROF_SOCK_QUEUED_OP_READ;
    }
}

static int
Win32NetworkQueueReceive(win32_mnet_connection *SocketConnection)
{
    SocketConnection->Operation = MPROF_SOCK_QUEUED_OP_READ;
    SocketConnection->BytesReceived = 0;

    SocketConnection->DataBuf.len = MPROF_SOCK_BUFFER_SIZE;
    SocketConnection->DataBuf.buf = SocketConnection->Buffer;

    DWORD Flags;
    if (WSARecv(SocketConnection->Socket, &(SocketConnection->DataBuf), 1, NULL, &Flags, &(SocketConnection->Overlapped), NULL) == SOCKET_ERROR)
    {
        if (WSAGetLastError() != ERROR_IO_PENDING)
        {
            Win32OutputWSAErrorCode("WSARecv Failed.");
            return 0;
        }
    }

    return 1;
}

// TODO(Oskar): Ofcourse we later want to pass in a buffer here.
static int
Win32NetworkQueueSend(win32_mnet_connection *SocketConnection)
{
    SocketConnection->Operation = MPROF_SOCK_QUEUED_OP_SEND;

    SocketConnection->DataBuf.buf = SocketConnection->Buffer + SocketConnection->BytesSent;
    SocketConnection->DataBuf.len = SocketConnection->BytesReceived - SocketConnection->BytesSent;

    if (WSASend(SocketConnection->Socket, &(SocketConnection->DataBuf), 1, NULL, 0, &(SocketConnection->Overlapped), NULL) == SOCKET_ERROR)
    {
        if (WSAGetLastError() != ERROR_IO_PENDING)
        {
            Win32OutputWSAErrorCode("WSASend Failed.");
            return 0;
        }
    }

    return 1;
}

static int
Win32NetworkListen(win32_mnet_state *State)
{
    if ((State->LastAcceptSocket = WSASocketW(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED)) == INVALID_SOCKET)
    {
        Win32OutputWSAErrorCode("Failed to create Accepting socket.");
        WSACleanup();
        return -1;
    }

    ZeroMemory(&State->ListenOverlapped, sizeof(OVERLAPPED));
    if ((State->EventArray[0] = State->ListenOverlapped.hEvent = WSACreateEvent()) == WSA_INVALID_EVENT)
    {
        Win32OutputWSAErrorCode("Failed to create new event object.");
        return -1;
    }

    State->NumberOfEvents = 1;
    
    CHAR AcceptBuffer[2 * (sizeof(SOCKADDR_IN) + 16)];
    DWORD Bytes = 0;
    if (State->AcceptEx(State->ListenSocket, State->LastAcceptSocket, AcceptBuffer, 0,
                        sizeof(SOCKADDR_IN) + 16, sizeof(SOCKADDR_IN) + 16, 
                        &Bytes, &State->ListenOverlapped) == FALSE)
    {
        if (WSAGetLastError() != ERROR_IO_PENDING)
        {
            Win32OutputWSAErrorCode("Initial AcceptEx failed.");
            return -1;
        }
    }

    win32_mnet_connection *SocketConnection;
    DWORD BytesTransferred;
    DWORD Flags;
    while(TRUE)
    {   
        DWORD WSAWaitResult = WSAWaitForMultipleEvents(State->NumberOfEvents, State->EventArray, FALSE, WSA_INFINITE, FALSE);
        if (WSAWaitResult == WSA_WAIT_FAILED)
        {
            Win32OutputWSAErrorCode("WSAWait Failed.");
            return 0;
        }

        DWORD EventIndex = WSAWaitResult - WSA_WAIT_EVENT_0;
        
        // NOTE(Oskar): Connection happened to the listening socket.
        if ((EventIndex) == 0)
        {
            if (WSAGetOverlappedResult(State->ListenSocket, &(State->ListenOverlapped), &BytesTransferred, FALSE, &Flags) == FALSE)
            {
                Win32OutputWSAErrorCode("Failed to get overlapped result for incoming connection.");
                return -1;
            }

            // NOTE(Oskar): Check if too many connections
            if (State->NumberOfEvents > WSA_MAXIMUM_WAIT_EVENTS)
            {
                closesocket(State->LastAcceptSocket);
                continue;
            }
            else
            {
                if ((State->Connections[State->NumberOfEvents] = (win32_mnet_connection *) HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(win32_mnet_connection))) == NULL)
                {
                    Win32OutputErrorCode("Call to global alloc failed.");
                    return -1;
                }

                win32_mnet_connection *TargetConnection = State->Connections[State->NumberOfEvents];
                _Win32SocketConnectionInitialize(TargetConnection, State->LastAcceptSocket);
                if ((TargetConnection->Overlapped.hEvent = State->EventArray[State->NumberOfEvents] = WSACreateEvent()) == WSA_INVALID_EVENT)
                {
                    Win32OutputWSAErrorCode("Failed to create new event object.");
                    return -1;
                }

                // NOTE(Oskar): Queue a new Recv op on the newly accepted socket.
                if (Win32NetworkQueueReceive(TargetConnection) == 0)
                {
                    return -1;
                }

                State->NumberOfEvents++;
            }

            // NOTE(Oskar): Create new socket and post another AcceptEx operation
            if ((State->LastAcceptSocket = WSASocketW(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED)) == INVALID_SOCKET)
            {
                Win32OutputWSAErrorCode("Failed to create Accepting socket.");
                return -1;
            }

            WSAResetEvent(State->EventArray[0]);
            ZeroMemory(&State->ListenOverlapped, sizeof(OVERLAPPED));
            State->ListenOverlapped.hEvent = State->EventArray[0];

            if (State->AcceptEx(State->ListenSocket, State->LastAcceptSocket, AcceptBuffer, 0,
                        sizeof(SOCKADDR_IN) + 16, sizeof(SOCKADDR_IN) + 16, 
                        &Bytes, &State->ListenOverlapped) == FALSE)
            {
                if (WSAGetLastError() != ERROR_IO_PENDING)
                {
                    Win32OutputWSAErrorCode("Queueing new AcceptEx failed.");
                    return -1;
                }
            }

            continue;
        }

        SocketConnection = State->Connections[EventIndex];
        WSAResetEvent(State->EventArray[EventIndex]);

        BOOL OverlappedResult = WSAGetOverlappedResult(SocketConnection->Socket, &(SocketConnection->Overlapped), &BytesTransferred, FALSE, &Flags);
        if (OverlappedResult == FALSE || BytesTransferred == 0)
        {
            printf("Closing socket %d\n", (int)SocketConnection->Socket);

            if (closesocket(SocketConnection->Socket) == SOCKET_ERROR)
            {
                Win32OutputWSAErrorCode("Call to closesocket Failed.");
            }

            HeapFree(GetProcessHeap(), 0, SocketConnection);
            WSACloseEvent(State->EventArray[EventIndex]);
            
            // NOTE(Oskar): Move event and socket handles to the back of their respective arrays.        
            if (EventIndex + 1 != State->NumberOfEvents)
            {
                for (DWORD Index = EventIndex; Index < State->NumberOfEvents; Index++)
                {
                    State->EventArray[Index] = State->EventArray[Index + 1];   
                    State->Connections[Index] = State->Connections[Index + 1];
                }
            }

            State->NumberOfEvents--;
            continue;
        }

        // NOTE(Oskar): Here we check what the last queued operation was and handle echo logic.
        Win32NetworkUpdateConnectionState(SocketConnection, BytesTransferred);

        // NOTE(Oskar): Send with WSASend. WSASend doesn't guarantee sending everything in one go
        // so we continue sending untill all bytes are sent.
        if (SocketConnection->Operation == MPROF_SOCK_QUEUED_OP_SEND)
        {
            ZeroMemory(&(SocketConnection->Overlapped), sizeof(WSAOVERLAPPED));
            SocketConnection->Overlapped.hEvent = State->EventArray[EventIndex];

            if (Win32NetworkQueueSend(SocketConnection) == 0)
            {
                return 0;
            }
        }
        else if (SocketConnection->Operation == MPROF_SOCK_QUEUED_OP_READ)
        {
            // NOTE(Oskar): No more byte to send so we continue with WSARecv.
            ZeroMemory(&(SocketConnection->Overlapped), sizeof(WSAOVERLAPPED));
            SocketConnection->Overlapped.hEvent = State->EventArray[EventIndex];

            if (Win32NetworkQueueReceive(SocketConnection) == 0)
            {
                return 0;
            }
        }
        else
        {
            // NOOP
        }
    }

    return 0;
}

win32_sock.h

#define MPROF_SOCK_DEFAULT_PORT "27015"
#define MPROF_SOCK_BUFFER_SIZE 4096
#define MPROF_SOCK_MAX_THREAD 4
#define MPROF_SOCK_MAX_IO_REQUESTS 10
#define MPROF_SOCK_DEFAULT_BACKLOG 5

typedef enum _win32_mnet_queued_op
{
    MPROF_SOCK_QUEUED_OP_NONE,
    MPROF_SOCK_QUEUED_OP_READ,
    MPROF_SOCK_QUEUED_OP_SEND,
} win32_mnet_queued_op;

typedef struct _win32_mnet_connection 
{
   CHAR Buffer[MPROF_SOCK_BUFFER_SIZE];
   WSABUF DataBuf;
   SOCKET Socket;
   WSAOVERLAPPED Overlapped;
   
   win32_mnet_queued_op Operation;
   DWORD BytesSent;
   DWORD BytesReceived;
} win32_mnet_connection;

typedef struct _win32_mnet_state
{
    struct addrinfo *AddressInfo;
    WSAOVERLAPPED ListenOverlapped;
    SOCKET ListenSocket;
    SOCKET LastAcceptSocket;

    DWORD NumberOfEvents;
    WSAEVENT EventArray[WSA_MAXIMUM_WAIT_EVENTS];

    win32_mnet_connection *Connections[WSA_MAXIMUM_WAIT_EVENTS];

    LPFN_ACCEPTEX AcceptEx;    
    BOOL Initialized;
} win32_mnet_state;