WASM module in C demonstrating using linear memory to share structures for sending and receiving HTTP requests/responses

demo-unit.c

/*
 * WIP/PROTOTYPE: Caveat emptor. If it breaks, you get to keep all the pieces.
 *
 * Compile with:
 *
 * clang -Wall -Wextra -fno-strict-aliasing --target=wasm32-wasi --sysroot=../wasi-sysroot -Wl,--no-entry,--export=__heap_base,--export=__data_end,--export=malloc,--export=free,--stack-first,-z,stack-size=$((8*1024*1024)),--initial-memory=$(((4096*1024*1024)-65536)) -mexec-model=reactor -o demo-unit.wasm demo-unit.c
 *
 * Download the wasi-sysroot tarball from https://github.com/WebAssembly/wasi-sdk/releases
 */

#define _GNU_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>

#include <fcntl.h>
#include <unistd.h>
#include <errno.h>

typedef uint64_t u64;
typedef int64_t  s64;
typedef uint32_t u32;
typedef int32_t  s32;
typedef uint16_t u16;
typedef int16_t  s16;
typedef uint8_t   u8;
typedef int8_t    s8;

#ifndef __unused
#define __unused                __attribute__((unused))
#endif
#ifndef __maybe_unused
#define __maybe_unused          __unused
#endif
#ifndef __always_unused
#define __always_unused         __unused
#endif

struct hdr_field {
        u32 name_offs;
        u32 name_len;
        u32 value_offs;
        u32 value_len;
};

struct req {
        u32 method_offs;
        u32 method_len;
        u32 version_offs;
        u32 version_len;
        u32 path_offs;
        u32 path_len;
        u32 query_offs;
        u32 query_len;
        u32 remote_offs;
        u32 remote_len;
        u32 local_addr_offs;
        u32 local_addr_len;
        u32 local_port_offs;
        u32 local_port_len;
        u32 server_name_offs;
        u32 server_name_len;

        u32 content_offs;
        u32 content_len;
        u32 content_sent;
        u32 total_content_sent;

        u32 request_size;

        u32 nr_fields;

        u32 tls;

        struct hdr_field fields[];
};

struct resp {
        u32 size;

        u8 data[];
};

struct resp_hdr {
        u32 nr_fields;

        struct hdr_field fields[];
};

static size_t total_response_sent;

static u8 *request_buf;

__attribute__((import_module("env"), import_name("nxt_wasm_send_headers")))
void nxt_wasm_send_headers(u32 offset);
__attribute__((import_module("env"), import_name("nxt_wasm_send_response")))
void nxt_wasm_send_response(u32 offset);

__attribute__((export_name("wasm_request_end_handler")))
void wasm_request_end_handler(void)
{
        if (!request_buf)
                return;

        free(request_buf);
        request_buf = NULL;
}

__attribute__((export_name("wasm_free_handler")))
void wasm_free_handler(u32 addr)
{
        free((void *)addr);
}

__attribute__((export_name("wasm_malloc_handler")))
u32 wasm_malloc_handler(size_t size)
{
        return (u32)malloc(size);
}

static void send_headers(u8 *addr, const char *ct, size_t len)
{
        struct resp_hdr *rh;
        char clen[32];
        u8 *p;
        static const u32 hdr_offs = 0;

        rh = (struct resp_hdr *)addr;

#define SET_HDR_FIELD(idx, name, val) \
        do { \
                rh->fields[idx].name_offs = p - addr; \
                rh->fields[idx].name_len = strlen(name); \
                p = mempcpy(p, name, rh->fields[idx].name_len); \
                rh->fields[idx].value_offs = p - addr; \
                rh->fields[idx].value_len = strlen(val); \
                p = mempcpy(p, val, rh->fields[idx].value_len); \
        } while (0)

        rh->nr_fields = 2;
        p = addr + sizeof(struct resp_hdr) +
                (rh->nr_fields * sizeof(struct hdr_field));

        SET_HDR_FIELD(0, "Content-Type", ct);
        snprintf(clen , sizeof(clen), "%lu", len);
        SET_HDR_FIELD(1, "Content-Length", clen);

        nxt_wasm_send_headers(hdr_offs);
}

static int upload_reflector(u8 *addr, size_t mem_size)
{
        size_t rsize = sizeof(struct resp);
        size_t write_bytes;
        struct req *req;
        struct resp *resp;
        const char *field;
        char ct[256];
        struct hdr_field *f;
        struct hdr_field *f_end;

        printf("==[WASM RESP]== %s:\n", __func__);

        resp = (struct resp *)addr;
        req = (struct req *)request_buf;

        printf("==[WASM RESP]== resp@%p\n", resp);
        printf("==[WASM RESP]== req@%p\n", req);
        printf("==[WASM RESP]== req->content_len    : %u\n", req->content_len);

        resp = (struct resp *)addr;

        /* Send headers */

        /* Try to set the Content-Type */
        f_end = req->fields + req->nr_fields;
        for (f = req->fields; f < f_end; f++) {
                field = (const char *)(u8 *)req + f->name_offs;

                if (strncasecmp(field, "Content-Type", 12) == 0) {
                        snprintf(ct, sizeof(ct), "%.*s", f->value_len,
                                 (u8 *)req + f->value_offs);
                        break;
                }

                field = NULL;
        }
        if (!field)
                sprintf(ct, "application/octet-stream");

        send_headers(addr, ct, req->content_len);

        do {
                write_bytes = req->content_len - total_response_sent;
                if (write_bytes > mem_size - rsize)
                        write_bytes = mem_size - rsize;

                printf("==[WASM RESP]== write_bytes         : %lu\n",
                       write_bytes);
                printf("==[WASM RESP]== req->content_len    : %u\n",
                       req->content_len);
                printf("==[WASM RESP]== total_response_sent : %lu\n",
                       total_response_sent);

                printf("==[WASM RESP]== Copying (%lu) bytes of data from "
                       "[%p+%lx] to [%p]\n", write_bytes, req,
                       req->content_offs + total_response_sent, resp->data);
                memcpy(resp->data,
                       (u8 *)req + req->content_offs + total_response_sent,
                       write_bytes);

                total_response_sent += write_bytes;
                resp->size = write_bytes;
                printf("==[WASM RESP]== resp->size          : %u\n",
                       resp->size);

                nxt_wasm_send_response(0);
        } while (write_bytes > 0);

        if (total_response_sent == req->content_len) {
                printf("==[WASM RESP]== All data sent. Cleaning up...\n");
                total_response_sent = 0;

                free(request_buf);
                request_buf = NULL;
        }

        return 0;
}

static int echo_request(u8 *addr, size_t mem_size __unused)
{
        u8 *p;
        const char *method;
        struct req *req;
        struct resp *resp;
        struct hdr_field *hf;
        struct hdr_field *hf_end;
        static const int resp_offs = 4096;

        printf("==[WASM RESP]== %s:\n", __func__);

        /*
         * For convenience, we will return our headers at the start
         * of the shared memory so leave a little space (resp_offs)
         * before storing the main response.
         *
         * send_headers() will return the start of the shared memory,
         * echo_request() will return the start of the shared memory
         * plus resp_offs.
         */
        resp = (struct resp *)(addr + resp_offs);

        req = (struct req *)request_buf;

#define BUF_ADD(name, member) \
        do { \
                p = mempcpy(p, name, strlen(name)); \
                p = mempcpy(p, (u8 *)req + req->member##_offs, req->member##_len); \
                p = mempcpy(p, "\n", 1); \
        } while (0)

#define BUF_ADD_HF() \
        do { \
                p = mempcpy(p, (u8 *)req + hf->name_offs, hf->name_len); \
                p = mempcpy(p, " = ", 3); \
                p = mempcpy(p, (u8 *)req + hf->value_offs, hf->value_len); \
                p = mempcpy(p, "\n", 1); \
        } while (0)

        p = resp->data;

        p = mempcpy(p, "Welcome to WebAssembly on Unit!\n\n", 33);

        p = mempcpy(p, "[Request Info]\n", 15);
        BUF_ADD("REQUEST_PATH = ", path);
        BUF_ADD("METHOD       = ", method);
        BUF_ADD("VERSION      = ", version);
        BUF_ADD("QUERY        = ", query);
        BUF_ADD("REMOTE       = ", remote);
        BUF_ADD("LOCAL_ADDR   = ", local_addr);
        BUF_ADD("LOCAL_PORT   = ", local_port);
        BUF_ADD("SERVER_NAME  = ", server_name);

        p = mempcpy(p, "\n[Request Headers]\n", 19);
        hf_end = req->fields + req->nr_fields;
        for (hf = req->fields; hf < hf_end; hf++)
                BUF_ADD_HF();

        method = (char *)req + req->method_offs;
        if (memcmp(method, "POST", req->method_len) == 0 ||
            memcmp(method, "PUT", req->method_len) == 0) {
                p = mempcpy(p, "\n[", 2);
                p = mempcpy(p, method, req->method_len);
                p = mempcpy(p, " data]\n", 7);
                p = mempcpy(p, (u8 *)req + req->content_offs, req->content_len);
                p = mempcpy(p, "\n", 1);
        }

        p = memcpy(p, "\0", 1);

        resp->size = p - resp->data;

        send_headers(addr, "text/plain", resp->size);

#if 0
        {
                static const char f[] = "/tmp/wasm-test";
                int fd = open(f, O_TRUNC|O_CREAT|O_WRONLY, 0666);
                if (fd == -1) {
                        printf("EEEE Couldn't open \"%s\": %s\n", f,
                                strerror(errno));
                }
                close(fd);
        }
#endif

        free(request_buf);
        /*
         * NULL out request_buf to signify the end of the request and
         * that the next call to wasm_request_handler() will be the
         * start of a new request.
         */
        request_buf = NULL;

        nxt_wasm_send_response(resp_offs);

        return 0;
}

__attribute__((export_name("wasm_request_handler")))
int wasm_request_handler(u8 *addr, size_t mem_size)
{
        struct req *req = (struct req *)addr;
        struct req *rb = (struct req *)request_buf;

        printf("==[WASM REQ]== %s:\n", __func__);

        /*
         * Do something with req. I.e check what path was requested and
         * set the response_worker function appropriately
         *
         * This function _may_ be called multiple times during a single
         * request if there is a large amount of data to transfer.
         *
         * req->content_len contains the overall size of the POST/PUT
         * data.
         * req->content_sent shows how much of the body content has been
         * in _this_ request.
         * req->total_content_sent shows how much of it has been sent in
         * total.
         * req->content_offs is the offset in the passed in memory where
         * the body content starts.
         *
         * For new requests req->request_size shows the total size of
         * _this_ request, incl the req structure itself.
         * For continuation requests, req->request_size is just the amount
         * of new content, i.e req->content_sent
         *
         * When req->content_len == req->total_content_sent, that's the end
         * of that request.
         */

        if (!request_buf) {
                /*
                 * Just allocate memory for the total amount of data we
                 * expect to get, this includes the request structure
                 * itself as well as any body content.
                 */
                printf("==[WASM REQ]== malloc(%u)\n",
                       req->content_offs + req->content_len);
                request_buf = malloc(req->content_offs + req->content_len);

                /*
                 * Regardless of how much memory we allocated above, here
                 * we only want to copy the amount of data we actually
                 * received in this request.
                 */
                printf("==[WASM REQ]== req->request_size : %u\n",
                       req->request_size);
                memcpy(request_buf, addr, req->request_size);

                rb = (struct req *)request_buf;
                printf("==[WASM REQ]== rb@%p\n", rb);
                printf("==[WASM REQ]== request_buf@%p\n", request_buf);
                printf("==[WASM REQ]== rb->content_offs : %u\n",
                       rb->content_offs);
                printf("==[WASM REQ]== rb->content_len  : %u\n",
                       rb->content_len);
                printf("==[WASM REQ]== rb->content_sent : %u\n",
                       rb->content_sent);
                printf("==[WASM REQ]== rb->request_size : %u\n",
                       rb->request_size);
        } else {
                memcpy(request_buf + rb->request_size, addr + req->content_offs,
                       req->request_size);

                printf("==[WASM REQ +]== req->content_offs : %u\n",
                       req->content_offs);
                printf("==[WASM REQ +]== req->content_sent : %u\n",
                       req->content_sent);
                printf("==[WASM REQ +]== req->request_size : %u\n",
                       req->request_size);

                rb->content_sent = req->content_sent;
                rb->total_content_sent = req->total_content_sent;
        }

        if (rb->total_content_sent != rb->content_len)
                return 0;

        printf("==[WASM REQ]== Looking up path (%.*s)\n",
               rb->path_len, (const char *)(u8 *)rb + rb->path_offs);
        if (strncmp((const char *)(u8 *)rb + rb->path_offs, "/upload", 7) == 0)
                upload_reflector(addr, mem_size);
        else
                echo_request(addr, mem_size);

        return 0;
}

The req structure contains a flexible array member (FAM) that represents the http headers sent by the client. The actual request data nd any POST data are stored after this FAM.

This returns something like the following

$ curl -X POST -d "Hello World" http://localhost:8080/wasmtime/?q=test
Welcome to WebAssembly on Unit!

[Request Info]
REQUEST_PATH = /wasmtime/?q=test
METHOD       = POST
VERSION      = HTTP/1.1
QUERY        = q=test
REMOTE       = ::1
LOCAL_ADDR   = ::1
LOCAL_PORT   = 8080
SERVER_NAME  = localhost

[Request Headers]
Host = localhost:8080
User-Agent = curl/8.0.1
Accept = */*
Content-Length = 11
Content-Type = application/x-www-form-urlencoded

[POST data]
Hello World
$

Now for a test to show uploading a file that is larger than the shared memory (32Mib) between the language module and the WASM module.

$ ls -oh Music/Solar\ Fields\ -\ Fiat\ Lux\ -\ Remastered.flac 
-rw-r--r-- 1 andrew 58M Mar  9  2017 'Music/Solar Fields - Fiat Lux - Remastered.flac'
$ curl -v -X POST --data-binary @Music/Solar\ Fields\ -\ Fiat\ Lux\ -\ Remastered.flac -H "Content-Type: audio/flac" http://localhost:8080/upload_reflector/ -o wasm-test.dat
Note: Unnecessary use of -X or --request, POST is already inferred.
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 127.0.0.1:8080...
* connect to 127.0.0.1 port 8080 failed: Connection refused
*   Trying [::1]:8080...
* Connected to localhost (::1) port 8080 (#0)
> POST /upload_reflector/ HTTP/1.1
> Host: localhost:8080
> User-Agent: curl/8.0.1
> Accept: */*
> Content-Type: audio/flac
> Content-Length: 60406273
> Expect: 100-continue
> 
* Done waiting for 100-continue
  0 57.6M    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0} [65536 bytes data]
* We are completely uploaded and fine
< HTTP/1.1 200 OK
< Content-Type: audio/flac
< Content-Length: 60406273
< Server: Unit/1.31.0
< Date: Thu, 08 Jun 2023 18:12:29 GMT
< 
{ [32768 bytes data]
100  115M  100 57.6M  100 57.6M  44.8M  44.8M  0:00:01  0:00:01 --:--:-- 89.7M
* Connection #0 to host localhost left intact
$ ls -oh wasm-test.dat 
-rw-r--r-- 1 andrew 58M Jun  8 19:12 wasm-test.dat
$ sha1sum Music/Solar\ Fields\ -\ Fiat\ Lux\ -\ Remastered.flac wasm-test.dat 
ef5c9c228544b237022584a8ac4612005cd6263e  Music/Solar Fields - Fiat Lux - Remastered.flac
ef5c9c228544b237022584a8ac4612005cd6263e  wasm-test.dat
$