C shim library for llama.cpp

llama-red.c

/*
** llama-red.c -- minimal extern-C cdecl shim over llama.cpp for Red/System
**
** llama.h passes its config structs by value (llama_model_params,
** llama_context_params, llama_sampler_chain_params, llama_batch). Red/System
** CAN pass structs by value (the `value` keyword), but mirroring these large,
** version-volatile layouts field-for-field on the Red side would break on
** every llama.cpp upgrade -- so they stay behind pointer-and-scalar entry
** points here. One opaque handle carries model+ctx+sampler.
**
** Build (static, for Red's -s linker): see build-shim.cmd -- cl /MT /O1,
** NO /GL (Red's COFF reader rejects LTCG bitcode), then lib-bundled with
** llama.lib + ggml*.lib into llama-red.lib.
*/

#include "llama.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef _WIN32
#include <windows.h>
#else
#include <unistd.h>
#endif

/* 0 or negative thread count = auto: half the online logical cores,
   at least 1 -- oversubscribing ggml's spin-waits collapses throughput
   (measured 8 threads on a 4-vCPU WSL: 68x SLOWER than 1 thread). */
static int lr_auto_threads(int n) {
    long nc;
    if (n > 0) return n;
#ifdef _WIN32
    { SYSTEM_INFO si; GetSystemInfo(&si); nc = (long)si.dwNumberOfProcessors; }
#else
    nc = sysconf(_SC_NPROCESSORS_ONLN);
#endif
    if (nc < 2) return 1;
    return (int)(nc / 2);
}

#ifdef _WIN32
#define LR_API __declspec(dllexport)   /* harmless in the static build; keeps the DLL option open */
#else
#define LR_API
#endif

/*
** Exception guard: llama's tokenizer/unicode layer can throw (std::regex,
** wstring conversions); an exception must NEVER unwind past this extern-C
** boundary into Red/System frames (no unwind tables there -> terminate).
** Compile as C++ (g++ -x c++ / cl /TP) to arm the guards; building as
** plain C leaves them empty (Windows/SEH tolerated that historically).
*/
#ifdef __cplusplus
#include <exception>
#define LR_TRY       try {
#define LR_CATCH(v)  } catch (const std::exception& e) { set_err(e.what()); return (v); } \
                     catch (...) { set_err("unknown C++ exception"); return (v); }
extern "C" {
#else
#define LR_TRY
#define LR_CATCH(v)
#endif

static char lr_err[512];

static void set_err(const char* msg) {
    snprintf(lr_err, sizeof lr_err, "%s", msg);
}

typedef struct lr_llm {
    struct llama_model*        model;
    struct llama_context*      ctx;
    const struct llama_vocab*  vocab;
    struct llama_sampler*      smpl;
    int                        n_past;    /* tokens already decoded */
} lr_llm;

/* ---- logging: llama is chatty on stderr by default ---- */

static void lr_null_logger(enum ggml_log_level level, const char* text, void* user) {
    (void)level; (void)text; (void)user;
}

LR_API const char* lr_last_error(void) {
    return lr_err;
}

LR_API void lr_quiet(void) {
    llama_log_set(lr_null_logger, NULL);
}

LR_API void lr_init(void) {
#ifdef _WIN32
    SetConsoleOutputCP(CP_UTF8);       /* UTF-8 pieces print correctly */
#endif
    llama_backend_init();
}

LR_API const char* lr_system_info(void) {
    return llama_print_system_info();  /* proves which SIMD paths are live */
}

/* ---- lifecycle ---- */

/* temp is a double on purpose: Red/System's ARM backend mis-stages a
   float32-cast argument (raw 8-byte double pushed, 4 popped) which leaves
   SP misaligned by 4 for the whole callee subtree -- AAPCS requires 8.
   A double argument keeps every layer (align-prolog calc, push, pop) in
   agreement, and the narrowing to float happens safely inside. */
LR_API lr_llm* lr_load(const char* path, int n_ctx, int n_threads, double temp, int seed) {
    LR_TRY
    lr_llm* h = (lr_llm*)calloc(1, sizeof(lr_llm));
    if (!h) { set_err("out of memory"); return NULL; }

    struct llama_model_params mparams = llama_model_default_params();
    mparams.use_mmap = true;           /* file-backed weights -- the 32-bit friend */

    h->model = llama_model_load_from_file(path, mparams);
    if (!h->model) {
        set_err("model load failed (path/format/memory)");
        free(h);
        return NULL;
    }
    h->vocab = llama_model_get_vocab(h->model);

    struct llama_context_params cparams = llama_context_default_params();
    n_threads = lr_auto_threads(n_threads);
    cparams.n_ctx           = n_ctx;
    cparams.n_batch         = n_ctx;   /* whole prompt in one decode call */
    cparams.n_threads       = n_threads;
    cparams.n_threads_batch = n_threads;

    h->ctx = llama_init_from_model(h->model, cparams);
    if (!h->ctx) {
        set_err("context creation failed (n_ctx too large for address space?)");
        llama_model_free(h->model);
        free(h);
        return NULL;
    }

    struct llama_sampler_chain_params sparams = llama_sampler_chain_default_params();
    h->smpl = llama_sampler_chain_init(sparams);
    if (temp <= 0.0f) {
        llama_sampler_chain_add(h->smpl, llama_sampler_init_greedy());
    } else {
        llama_sampler_chain_add(h->smpl, llama_sampler_init_top_k(40));
        llama_sampler_chain_add(h->smpl, llama_sampler_init_top_p(0.95f, 1));
        llama_sampler_chain_add(h->smpl, llama_sampler_init_temp((float)temp));
        llama_sampler_chain_add(h->smpl, llama_sampler_init_dist((uint32_t)seed));
    }
    return h;
    LR_CATCH(NULL)
}

LR_API void lr_free(lr_llm* h) {
    if (!h) return;
    if (h->smpl)  llama_sampler_free(h->smpl);
    if (h->ctx)   llama_free(h->ctx);
    if (h->model) llama_model_free(h->model);
    free(h);
}

LR_API int lr_n_ctx(lr_llm* h) {
    return h ? (int)llama_n_ctx(h->ctx) : 0;
}

/* ---- prompt ingestion ----
** Applies the model's chat template (single user turn) when use_template,
** tokenizes, decodes the whole prompt. Returns prompt token count, -1 err.
*/
LR_API int lr_start(lr_llm* h, const char* user_text, int use_template) {
    LR_TRY
    const char*  text = user_text;
    char*        templated = NULL;
    llama_token* tokens = NULL;

    if (!h) { set_err("null handle"); return -1; }

    if (use_template) {
        /* chatml wrap (Qwen & friends), built by hand: on the Linux static
           link llama_chat_apply_template aborts internally (single known
           divergence -- every other C++ path passes; root-cause fixture is
           on the backlog). The hand wrap also drops a template-engine
           dependency the demo never needed. */
        int cap = (int)strlen(user_text) + 96;
        templated = (char*)malloc(cap);
        if (!templated) { set_err("out of memory"); return -1; }
        snprintf(templated, cap,
            "<|im_start|>user\n%s<|im_end|>\n<|im_start|>assistant\n", user_text);
        text = templated;
    }

    int n_max = (int)strlen(text) + 64;
    tokens = (llama_token*)malloc(n_max * sizeof(llama_token));
    if (!tokens) { set_err("out of memory"); free(templated); return -1; }

    int n = llama_tokenize(h->vocab, text, (int)strlen(text), tokens, n_max,
                           /*add_special*/ !use_template, /*parse_special*/ true);
    if (n < 0) { set_err("tokenize failed"); free(tokens); free(templated); return -1; }
    if (n >= lr_n_ctx(h)) { set_err("prompt longer than n_ctx"); free(tokens); free(templated); return -1; }

    struct llama_batch batch = llama_batch_get_one(tokens, n);
    if (llama_decode(h->ctx, batch) != 0) {
        set_err("decode failed on prompt");
        free(tokens); free(templated);
        return -1;
    }
    h->n_past = n;

    free(tokens);
    free(templated);
    return n;
    LR_CATCH(-1)
}

/* ---- generation ----
** Samples one token, appends its text to buf (UTF-8, NOT zero-padded
** beyond len), decodes it. Returns piece byte length (>= 0),
** -1 = end of generation (EOG token), -2 = context full, -3 = error.
*/
LR_API int lr_next(lr_llm* h, char* buf, int buf_size) {
    LR_TRY
    if (!h) { set_err("null handle"); return -3; }
    if (h->n_past >= lr_n_ctx(h) - 1) return -2;

    llama_token tok = llama_sampler_sample(h->smpl, h->ctx, -1);
    if (llama_vocab_is_eog(h->vocab, tok)) return -1;

    int len = llama_token_to_piece(h->vocab, tok, buf, buf_size, 0, /*special*/ false);
    if (len < 0) len = 0;              /* piece longer than buf: skip text, still decode */

    struct llama_batch batch = llama_batch_get_one(&tok, 1);
    if (llama_decode(h->ctx, batch) != 0) { set_err("decode failed"); return -3; }
    h->n_past += 1;
    return len;
    LR_CATCH(-3)
}

#ifdef __cplusplus
}   /* extern "C" */
#endif