A small program to proxy and record all traffic to a server.

verbose_proxy.c

/* ********************************************************
 * Copyright ©2024 Rundata Systems.  All rights reserved.
 * This project is licensed under the GPLv3 License. You
 * can find a copy of this license at:
 *    https://www.gnu.org/licenses/gpl-3.0.en.html
 *
 * More detail (1m read):
 * https://www.rundata.co.za/rundata/products/verbose_proxy
 *
 * Example usage (3m video):
 * https://www.youtube.com/watch?v=kpsFSY-G5F0
 *
 */

#warning TODO: 1. Port to Windows + MacOS
#warning TODO: 2. Implement MitM using user-supplied certificates

#define COPYRIGHT_BANNER   \
"        Copyright ©2024 Rundata Systems.  All rights reserved.\n"\
"        This project is licensed under the GPLv3 License. You\n"\
"        can find a copy of this license at:\n"\
"              https://www.gnu.org/licenses/gpl-3.0.en.html\n"\


/* ********************************************************
 * See the link to youtube for a short example of usage.
 *
 * I use this program in two vertically-maximised xterms
 * side-by-side on a separate workspace. Sometimes on a second
 * screen. One terminal does
 *    `tail -f verbose_server.txt`
 * and the other terminal does
 *    `tail -f * verbose_client.txt`.
 *
 * I find it useful to watch traffic when developing, instead of
 * using whatever devtools to look for the traffic when a problem
 * arises.
 *
 * Clicking around in the interface is slower than simply switching
 * workspaces and eyeballing the traffic scroll by.
 */


/* ********************************************************
 * Compiled with:
 *    gcc -W -Wall -Wextra -g verbose_proxy.c -o verbose_proxy
 *
 * The easiest way to execute the compile command above is by
 * copying and pasting it into the command-line.
 */

// Standard headers
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <inttypes.h>
#include <string.h>
#include <signal.h>
#include <errno.h>
#include <ctype.h>


// POSIX headers
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/select.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <pthread.h>

// Linux headers

// Project headers



/* ************************************************************************* *
 * TODO: At some point replace this with the library calls to
 * libnetcode if I want this to be portable to Windows.
 * Copied verbatim out of libnetcode ...
 */
int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags);
typedef int socket_t;
#define SAFETY_CHECK
#define NETCODE_INVALID_SOCKET  -1
#define NETCODE_SOCK_VALID(x) (x > 0)

char *netcode_util_sockaddr_to_str (const struct sockaddr *sa)
{
#define UNKNOWN_AF      ("Unknown Address Family")
   char *ret = NULL;
   if (!sa) {
      ret = calloc (1, 2);
      ret[0] = 0;
      return ret;
   }

   switch (sa->sa_family) {
      case AF_INET:
         if (!(ret = calloc (1, INET_ADDRSTRLEN + 1)))
            return NULL;
         inet_ntop (AF_INET, &(((struct sockaddr_in *)sa)->sin_addr),
                    ret, INET_ADDRSTRLEN);
         break;

      case AF_INET6:
         if (!(ret = calloc (1, INET6_ADDRSTRLEN + 1)))
            return NULL;
         inet_ntop (AF_INET6, &(((struct sockaddr_in6 *)sa)->sin6_addr),
                    ret, INET6_ADDRSTRLEN);
         break;

      default:
         if (!(ret = calloc (1, strlen (UNKNOWN_AF) + 1)))
            return NULL;
         strcpy (ret, UNKNOWN_AF);
         break;
   }

   return ret;
}
socket_t netcode_tcp_server (uint16_t port)
{
   /* ****************************************
    * 1. Call socket() to create a socket
    * 2. Call bind() to bind to a local address
    * 3. Call listen() to wait for incoming connection
    * 4. Call accept() to get the clients connection.
    */
   SAFETY_CHECK;
   struct sockaddr_in addr;

   addr.sin_family = AF_INET;
   addr.sin_addr.s_addr = INADDR_ANY;
   addr.sin_port = htons (port);
   socket_t fd = -1;
   if (port==0) {
      return -1;
   }
   fd = socket (AF_INET, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
   if (!(NETCODE_SOCK_VALID(fd))) {
      // NETCODE_UTIL_LOG ("socket() failed\n");
      return -1;
   }

   int optval = 1;
   if ((setsockopt (fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof optval)) !=0) {
      // Warn??? This is not really required anyway
   }

   if (bind (fd, (struct sockaddr *)&addr, sizeof addr)!=0) {
      // NETCODE_UTIL_LOG ("bind() failed\n");
      close (fd); fd = -1;
      return -1;
   }
   if (listen (fd, 1000)!=0) {
      // NETCODE_UTIL_LOG ("listen() failed\n");
      close (fd); fd = -1;
      return -1;
   }
   return fd;
}

socket_t netcode_tcp_accept (socket_t fd, uint32_t timeout_secs, char **addr, uint16_t *port)
{
   struct sockaddr_in ret;
   socklen_t retlen = sizeof ret;
   socket_t retval = -1;

   memset(&ret, 0xff, sizeof ret);

   struct timeval tv = { timeout_secs , 0 };
   fd_set fds[3];
   for (uint32_t i=0; i<sizeof fds/sizeof fds[0]; i++) {
      FD_ZERO (&fds[i]);
      FD_SET (fd, &fds[i]);
   }
   int r = select ((int)fd + 1, &fds[0], NULL, &fds[2], &tv);
   if (r==0) {
      return 0;
   }
   /* *******************************************************************
    * TODO: When migrating to linking with libnetcode, remember that
    * that library, by default, returns sockets in non-blocking mode.
    * The caller must set socket options to blocking.
    */
   // retval = accept4 (fd, (struct sockaddr *)&ret, &retlen, SOCK_NONBLOCK | SOCK_CLOEXEC);
   retval = accept4 (fd, (struct sockaddr *)&ret, &retlen, SOCK_CLOEXEC);
   if (retval <= 0) {
      return -1;
   }

   /* This should be performed by the caller on every accepted socket.
#ifdef OSTYPE_Darwin
   int optval = SO_NOSIGPIPE;
   if (setsockopt (retval, SOL_SOCKET, &optval, sizeof optval)!=0) {
      FPRINTF (stderr, "setsockopt(NOSIGPIPE) failure.\n");
      close (retval);
      return -1;
   }
#endif
   */

   if (addr) {
      *addr = netcode_util_sockaddr_to_str ((const struct sockaddr *)&ret);
   }

   if (port) {
      *port = ntohs (ret.sin_port);
   }
   return retval;
}

socket_t netcode_tcp_connect (const char *server, uint16_t port)
{
   /* ****************************************
    * 0. Resolve the server name.
    * 1. Call socket() to create a new socket.
    * 2. Call connect() to connect to a remote server.
    */
   SAFETY_CHECK;
   // Resolving server name
   struct hostent *serv_addr = gethostbyname (server);
   struct sockaddr_in addr;
   addr.sin_family = AF_INET;
   addr.sin_port = htons (port);
   addr.sin_addr.s_addr = inet_addr (server);

   if (!serv_addr) {
      return -1;
   }

   memcpy (&addr.sin_addr.s_addr, serv_addr->h_addr_list[0],
         sizeof addr.sin_addr.s_addr);

   // Creating socket endpoint
   socket_t fd = socket (AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
   if (!(NETCODE_SOCK_VALID(fd)))
      return -1;

   // Connecting endpoint to the server
   if (connect (fd, (struct sockaddr *)&addr, sizeof addr)!=0) {
      close (fd);
      return -1;
   }

   // Returning the connected fd to be used for reading/writing
   return fd;
}

/* ************************************************************************* */

#define FPRINTF(f,...)    do {\
   fprintf (f, "%s:%i in %s(): ", __FILE__, __LINE__, __func__);\
   fprintf (f, __VA_ARGS__);\
} while (0);


static volatile sig_atomic_t g_end = 0;
void sigh (int n)
{
   if (n == SIGINT) {
      g_end = 1;
   }
}


#define VERSION "0.0.5"

static char *sstrdup (const char *src)
{
   if (!src)
      return NULL;
   char *ret = NULL;
   size_t nbytes = strlen (src) + 1;
   if (!(ret = malloc (nbytes))) {
      FPRINTF (stderr, "OOM error allocating new string from [%s]\n", src);
      return NULL;
   }

   return strcpy (ret, src);
}

static bool smemicmp (const uint8_t *lhs, size_t nbytes, const uint8_t *rhs)
{
   for (size_t i=0; i<nbytes; i++) {
      if (tolower (*lhs++) != tolower (*rhs++)) {
         return false;
      }
   }
   return true;
}

/* Processor goes brrrrrrrr */
static uint8_t *sstrifind (uint8_t *src, size_t srclen,
                           const char *term, size_t termlen)
{
   if (!src || !term)
      return NULL;

   if (srclen < termlen)
      return NULL;

   uint8_t *ret = NULL;
   for (size_t i=0; i<(srclen - termlen + 1); i++) {
      if ((smemicmp (&src[i], termlen, (const uint8_t *)term)))
         return &src[i];
   }
   return ret;
}


struct thread_params_t {
   socket_t dst;
   socket_t src;
   FILE *outf;
   pthread_mutex_t *iomut;
};

static void thread_params_del (struct thread_params_t *p)
{
   if (!p)
      return;
   shutdown (p->src, SHUT_RDWR);
   shutdown (p->dst, SHUT_RDWR);
   close (p->src);
   close (p->dst);
   free (p);
}

static struct thread_params_t *thread_params_new (socket_t dst,
                                                  socket_t src,
                                                  pthread_mutex_t *iomut,
                                                  FILE *outf)
{
   struct thread_params_t *ret = calloc (1, sizeof *ret);
   if (!ret) {
      FPRINTF (stderr, "Failed to allocate memory for client [%i:%i]\n",
            dst, src);
      thread_params_del (ret);
      return NULL;
   }

   ret->dst = dst;
   ret->src = src;
   ret->outf = outf;
   ret->iomut = iomut;
   return ret;
}

static bool g_rawmode = false;

static void *proxyfd (void *params)
{
   struct thread_params_t *p = params;
   static const char str_ac[] = "accept-encoding";
   static const size_t str_aclen = (sizeof str_ac) - 1;

   static const size_t buflen = 1024 * 1024 * 8; // 8MB buffers
   uint8_t *buf = malloc (buflen * sizeof *buf);
   if (!buf) {
      FPRINTF (stderr, "Failed to allocate buffer\n");
      return NULL;
   }

   ssize_t nread;
   while ((nread = read (p->src, buf, buflen)) > 0) {

      if (g_rawmode) {
         uint8_t *ac = sstrifind(buf, nread, str_ac, str_aclen);
         if (ac) {
            memset (ac, 'X', str_aclen);
         }
      }

      pthread_mutex_lock (p->iomut);
      fwrite (buf, 1, nread, p->outf);
      fflush (p->outf);
      pthread_mutex_unlock (p->iomut);

      ssize_t index = 0;
      while (index < nread) {
         ssize_t nwrite = write (p->dst, &buf[index], nread - index);
         if (nwrite < 0) {
            // Not a very useful error message here - too much context
            // is missing. But, we can return an error and the user should
            // be able to determine which host or client was not written
            // to.
            FPRINTF (stderr, "Error writing to fd %i\n", p->dst);
            return false;
         }
         index += nwrite > 0 ? nwrite : 0;
      }
   }
   free (buf);
   thread_params_del (p);
   FPRINTF (stderr, "Thread ending\n");
   return NULL;
}


static bool parse_all_args (const char *lport, const char *server_url,
                            size_t *dst_lport, char **dst_host, size_t *dst_sport)
{
   bool error = true;

   free (*dst_host);
   *dst_host = NULL;

   if (!lport || !server_url) {
      FPRINTF (stderr, "Missing mandatory arguments:\n"
               "listen-port: [%s]\n"
               "server_url: [%s]\n",
               lport, server_url);
      goto cleanup;
   }

   if (!(*dst_host = sstrdup (server_url))) {
      FPRINTF (stderr, "Failed to make a copy of server_url [%s]\n", server_url);
      goto cleanup;
   }

   char *tmp = strchr (*dst_host, ':');
   if (!tmp) {
      FPRINTF (stderr, "No port specified in server_url [%s]\n", server_url);
      goto cleanup;
   }

   *tmp++ = 0;
   if ((sscanf (tmp, "%zu", dst_sport)) != 1) {
      FPRINTF (stderr, "Failed to parse server port [%s]\n", tmp);
      goto cleanup;
   }

   if ((sscanf (lport, "%zu", dst_lport)) != 1) {
      FPRINTF (stderr, "Failed to parse server port [%s]\n", lport);
      goto cleanup;
   }

   if (*dst_lport >= 0xffff || *dst_sport >= 0xffff) {
      FPRINTF (stderr, "Invalid port number specified\n"
               "listen-port: [%zu]\n"
               "server_url: [%zu]\n",
               *dst_lport, *dst_sport);
      goto cleanup;
   }


   error = false;
cleanup:
   if (error) {
      free (*dst_host);
      *dst_host = NULL;
   }

   return !error;
}

static bool thread_create (socket_t dst, socket_t src,
                           pthread_mutex_t *mut, FILE *outf)
{
   struct thread_params_t *params = thread_params_new (dst, src, mut, outf);
   if (!params) {
      FPRINTF (stderr, "Failed to create thread_params_t objects\n");
      thread_params_del (params);
      return false;
   }

   pthread_t tid;
   pthread_attr_t attr;
   pthread_attr_init (&attr);
   pthread_attr_setdetachstate (&attr, 1);

   if ((errno = pthread_create (&tid, &attr, proxyfd, params)) != 0) {
      FPRINTF (stderr, "Failed to create new thread: %m\n");
      return false;
   }

   return true;
}

static const char *g_helpmsg[] = {
"NAME",
"  Verbose Proxy: a program to proxy tcp text streams while capturing them.",
"",
"SYNPOSIS",
"  verbose_proxy [options] <listen-port> <server-url>",
"",
"DESCRIPTION",
"     <listen-port> must be a valid and free port on the local computer.",
"     <server-url> must be of the form hostname:port. No protocol specification",
"     prefix is allowed on the URL (protocol specification is not relevant",
"     as all the data is captured as text).",
"",
"OPTIONS",
"  -c <filename>     Capture data *from* the client to the file <filename>.",
"                    The default is verbose_client.txt",
"  -s <filename>     Capture data *from* the server to the file <filename>.",
"                    The default is verbose_server.txt",
"  -r                Turns on raw mode, in which every byte is printed exactly",
"                    as received. The default is to attempt to uncompressed HTTP",
"                    payloads.",
"  -h                Print this message and exit with zero.",
"  -v                Print version information and exit with zero.",
"",
"BUGS",
"  Most likely. Send bug reports to lee@rundata.co.za, with the subject",
"  'bug-report: verbose_proxy' or something similar.",
"",
NULL,
};

static const char *g_versionmsg[] = {
"Verbose Proxy " VERSION,
"",
COPYRIGHT_BANNER,
"https://www.rundata.co.za/rundata/products/verbose_proxy",
"",
NULL,
};

static void print_msg (const char **msg)
{
   for (size_t i=0; msg[i]; i++) {
      printf ("%s\n", msg[i]);
   }
}

int main (int argc, char **argv)
{
   int ret = EXIT_FAILURE;
   char *opt_cfname = sstrdup ("verbose_client.txt"),
        *opt_sfname = sstrdup ("verbose_server.txt");
   char *arg_lport = NULL;
   char *arg_server_url = NULL;

   char *addr = NULL;
   char *host = sstrdup ("");
   size_t lport = 0,
          sport = 0;

   FILE *client_outf = NULL;
   FILE *server_outf = NULL;

   pthread_mutex_t iomut_client;
   pthread_mutex_t iomut_server;

   socket_t listenfd = (socket_t)-1;

   if ((errno = pthread_mutex_init (&iomut_client, NULL)) != 0) {
      FPRINTF (stderr, "Failed to initialise client mutex: %m\n");
      goto cleanup;
   }

   if ((errno = pthread_mutex_init (&iomut_server, NULL)) != 0) {
      FPRINTF (stderr, "Failed to initialise server mutex: %m\n");
      goto cleanup;
   }


   // Parse all the options
   int argv_index = 0;
   for (argv_index=1; argv_index<argc && argv[argv_index]; argv_index++) {
      if ((argv[argv_index][0]) != '-') {
         break;
      }
      switch (argv[argv_index][1]) {
         case 'c':
            free (opt_cfname);
            opt_cfname = sstrdup (argv[++argv_index]);
            break;

         case 's':
            free (opt_sfname);
            opt_sfname = sstrdup (argv[++argv_index]);
            break;

         case 'r':
            g_rawmode = true;
            break;

         case 'h':
            print_msg (g_helpmsg);
            return EXIT_SUCCESS;

         case 'v':
            print_msg (g_versionmsg);
            return EXIT_SUCCESS;

         default:
            FPRINTF (stderr, "Unrecognised option flag '%s'\n", argv[argv_index]);
            goto cleanup;
      }
   }

   if (!opt_cfname || !opt_sfname) {
      FPRINTF (stderr, "Mangled command-line. Try -h\n");
      goto cleanup;
   }
   if (!(client_outf = fopen (opt_cfname, "w")) ||
         !(server_outf = fopen (opt_sfname, "w"))) {
      FPRINTF (stderr, "Failed to open output files\n"
               "[%s]=%p\n"
               "[%s]=%p\n",
               opt_cfname, client_outf,
               opt_sfname, server_outf);
      goto cleanup;
   }

   printf ("Verbose Proxy version %s\n", VERSION);
   printf ("Writing output to [%s] and [%s]\n", opt_cfname, opt_sfname);

   // Parse the mandatory arguments - argv_index points to the
   // listening-port argument
   if (!argv[argv_index] || !(arg_lport = sstrdup (argv[argv_index]))) {
      FPRINTF (stderr, "Missing a listening port number. Try -h\n");
      goto cleanup;
   }
   argv_index++;

   if (!argv[argv_index] || !(arg_server_url = sstrdup (argv[argv_index]))) {
      FPRINTF (stderr, "Missing a server URL. Try -h\n");
      goto cleanup;
   }

   if (!arg_lport || !arg_server_url) {
      FPRINTF (stderr, "Failed to read listening-port and server-url [%p:%p]\n",
               arg_lport, arg_server_url);
      goto cleanup;
   }

   if (!(parse_all_args (arg_lport, arg_server_url, &lport, &host, &sport))) {
      FPRINTF (stderr, "Failed to parse the command line arguments:"
               "[listening-port: %s]  "
               "[server-url: %s]\n",
               arg_lport, arg_server_url);
      goto cleanup;
   }

   // Install the signal handler to exit cleanly
   errno = 0;
   if ((signal (SIGINT, sigh)) == SIG_ERR) {
      FPRINTF (stderr, "Failed to install signal handler: %m\n");
      goto cleanup;
   }

   // Return errors on disconnected writes, don't raise the signal
    signal (SIGPIPE, SIG_IGN);

   // Start the listener
   errno = 0;
   listenfd = netcode_tcp_server ((uint16_t)lport);
   if (!(NETCODE_SOCK_VALID (listenfd))) {
      FPRINTF (stderr, "Failed to start listener on port %zu: %m\n", lport);
      goto cleanup;
   }

   int errcount = 0;
   uint16_t port = 0;
   while (g_end == 0 && errcount < 5) {
      free (addr);
      addr = NULL;
      errno = 0;
      socket_t clientfd = netcode_tcp_accept (listenfd, 1, &addr, &port);
      if (clientfd == 0) { // Timed out, ignore
         continue;
      }
      if (!(NETCODE_SOCK_VALID (clientfd))) {
         FPRINTF (stderr, "accept returned [%i]: accept() failed: %m\n", clientfd);
         sleep (1);
         errcount++;
         continue;
      }
      errcount = 0;

      FPRINTF (stdout, "Accepted client [%s:%u]\n", addr, port);

      socket_t serverfd = netcode_tcp_connect (host, sport);
      if (!(NETCODE_SOCK_VALID (serverfd))) {
         FPRINTF (stderr, "Failed to connect to [%s:%zu]\n", host, sport);
         break;
      }

      if (!(thread_create (clientfd, serverfd, &iomut_server, server_outf))) {
         FPRINTF (stderr, "Failed to create thread for server output [%s:%u]\n",
                  addr, port);
         raise (SIGINT);
         break;
      }

      if (!(thread_create (serverfd, clientfd, &iomut_client, client_outf))) {
         FPRINTF (stderr, "Failed to create thread for client output [%s:%u]\n",
                  addr, port);
         raise (SIGINT);
         break;
      }
   }

   // This is a memory leak, right here, when this point is reached while there
   // are still threads with connected hosts. Easy enough to store threadcount
   // and wait for it to reach zero, but why bother - we end the program at
   // this point.
   //    *shrug*
   ret = EXIT_SUCCESS;

cleanup:
   if (client_outf) {
      fclose (client_outf);
   }
   if (server_outf) {
      fclose (server_outf);
   }

   pthread_mutex_destroy (&iomut_client);
   pthread_mutex_destroy (&iomut_server);
   free (addr);
   free (host);
   free (opt_cfname);
   free (opt_sfname);
   free (arg_lport);
   free (arg_server_url);
   return ret;
}