Bash loadable builtins (enable -f): fast_kv + libexpanse-backed kv_trie — for shell-tips.com

README.md

Bash loadable builtins (enable -f)

Worked examples for How to Use Bash Loadable Builtins with enable -f on shell-tips.com. Both are C builtins that store and retrieve values in the shell's own process memory, with no subshell.

File	Storage	Needs
fast_kv.c	hand-rolled linked list (self-contained)	Bash C headers only
kv_trie.c	off-heap Expanse trie	Bash C headers + libexpanse

Build

# Bash C headers (both examples)
sudo apt-get install bash-builtins      # Debian/Ubuntu
sudo dnf install bash-devel             # Fedora/RHEL/AlmaLinux

make            # -> libfast_kv.so

kv_trie additionally links libexpanse, a modernized take on Judy arrays with a native, memory-safe C API:

# Debian/Ubuntu
echo "deb [trusted=yes] https://orieg.github.io/expanse/apt/ stable main" | sudo tee /etc/apt/sources.list.d/expanse.list
sudo apt-get update && sudo apt-get install -y libexpanse1 libexpanse-dev

make kv_trie    # -> libkv_trie.so (gcc ... -lexpanse)

Load and test

enable -f ./libfast_kv.so fast_kv
fast_kv set worker_threads 8
fast_kv get worker_threads      # -> 8

enable -f ./libkv_trie.so kv_trie
kv_trie set 192168001001 450
kv_trie get 192168001001        # -> 450

help fast_kv                    # registered docs
enable -d fast_kv               # unload

fast_kv.c

/* fast_kv.c - Minimal Bash loadable builtin */
#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "builtins.h"
#include "shell.h"
#include "bashgetopt.h"
#include "common.h"

/* Simple in-memory entry */
struct kv_entry {
    char key[64];
    long value;
    struct kv_entry *next;
};
static struct kv_entry *head = NULL;

int fast_kv_builtin(WORD_LIST *list) {
    if (list == NULL) {
        builtin_usage();
        return (EX_USAGE);
    }

    char *action = list->word->word;
    list = list->next;

    if (strcmp(action, "set") == 0) {
        if (!list || !list->next) {
            builtin_error("usage: fast_kv set <key> <value>");
            return (EXECUTION_FAILURE);
        }
        char *key = list->word->word;
        long val = atol(list->next->word->word);

        /* Update existing or prepend new */
        struct kv_entry *e = head;
        while (e) {
            if (strcmp(e->key, key) == 0) {
                e->value = val;
                return (EXECUTION_SUCCESS);
            }
            e = e->next;
        }

        struct kv_entry *new_entry = malloc(sizeof(struct kv_entry));
        strncpy(new_entry->key, key, sizeof(new_entry->key) - 1);
        new_entry->key[sizeof(new_entry->key) - 1] = '\0';
        new_entry->value = val;
        new_entry->next = head;
        head = new_entry;
        return (EXECUTION_SUCCESS);
    }

    if (strcmp(action, "get") == 0) {
        if (!list) {
            builtin_error("usage: fast_kv get <key>");
            return (EXECUTION_FAILURE);
        }
        char *key = list->word->word;
        struct kv_entry *e = head;
        while (e) {
            if (strcmp(e->key, key) == 0) {
                printf("%ld\n", e->value);
                return (EXECUTION_SUCCESS);
            }
            e = e->next;
        }
        return (EXECUTION_FAILURE);
    }

    builtin_error("unknown action '%s': use set or get", action);
    return (EX_USAGE);
}

/* Documentation displayed by the help builtin */
char *fast_kv_doc[] = {
    "Store and retrieve key-value pairs in shell memory.",
    "",
    "Usage: fast_kv set <key> <value>",
    "       fast_kv get <key>",
    "",
    "Manipulates in-memory key-value data directly without subshells.",
    (char *)NULL
};

/* Exported builtin descriptor */
struct builtin fast_kv_struct = {
    "fast_kv",                  /* Builtin command name */
    fast_kv_builtin,            /* Function pointer */
    BUILTIN_ENABLED,            /* Initial status */
    fast_kv_doc,                /* Documentation strings */
    "fast_kv set|get <key> [val]", /* Short usage synopsis */
    0                           /* Reserved */
};

kv_trie.c

/* kv_trie.c - Bash loadable builtin backed by libexpanse (native C API).
 *
 * Same idea as fast_kv.c, but keys and values live in an off-heap Expanse
 * trie instead of a hand-rolled linked list on the shell heap. This is the
 * pattern you want for high-cardinality data (100k+ keys).
 *
 * Install libexpanse (see https://github.com/orieg/expanse):
 *   Debian/Ubuntu : sudo apt-get install libexpanse1 libexpanse-dev
 *   Fedora/RHEL   : sudo dnf install libexpanse libexpanse-devel
 *
 *   gcc -fPIC -shared -I/usr/include/bash -I/usr/include/bash/include \
 *       -I/usr/include/bash/builtins -o libkv_trie.so kv_trie.c -lexpanse
 */
#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>

#include "builtins.h"
#include "shell.h"
#include "bashgetopt.h"
#include "common.h"

#include <expanse.h>

/* One ordered uint64 -> uint64 map, off-heap, for the shell's lifetime. */
static expanse_map_t *kv = NULL;

int kv_trie_builtin(WORD_LIST *list) {
    if (list == NULL) {
        builtin_usage();
        return (EX_USAGE);
    }
    if (kv == NULL && (kv = expanse_map_new()) == NULL) {
        builtin_error("kv_trie: cannot allocate map");
        return (EXECUTION_FAILURE);
    }

    char *action = list->word->word;
    list = list->next;

    if (strcmp(action, "set") == 0) {
        if (!list || !list->next) {
            builtin_error("usage: kv_trie set <key> <value>");
            return (EXECUTION_FAILURE);
        }
        uint64_t key = strtoull(list->word->word, NULL, 10);
        uint64_t val = strtoull(list->next->word->word, NULL, 10);
        expanse_map_insert(kv, key, val, NULL);   /* insert or overwrite */
        return (EXECUTION_SUCCESS);
    }

    if (strcmp(action, "get") == 0) {
        if (!list) {
            builtin_error("usage: kv_trie get <key>");
            return (EXECUTION_FAILURE);
        }
        uint64_t key = strtoull(list->word->word, NULL, 10);
        uint64_t val;
        if (!expanse_map_get(kv, key, &val))
            return (EXECUTION_FAILURE);            /* key not found */
        printf("%llu\n", (unsigned long long) val);
        return (EXECUTION_SUCCESS);
    }

    builtin_error("unknown action '%s': use set or get", action);
    return (EX_USAGE);
}

char *kv_trie_doc[] = {
    "Store and retrieve integer key/value pairs in an off-heap trie.",
    "",
    "Usage: kv_trie set <key> <value>",
    "       kv_trie get <key>",
    "",
    "Backed by libexpanse's native C API, not a shell heap allocation.",
    (char *)NULL
};

struct builtin kv_trie_struct = {
    "kv_trie",                      /* Builtin command name */
    kv_trie_builtin,                /* Function pointer */
    BUILTIN_ENABLED,                /* Initial status */
    kv_trie_doc,                    /* Documentation strings */
    "kv_trie set|get <key> [val]",  /* Short usage synopsis */
    0                               /* Reserved */
};

Makefile

# Build Bash loadable builtins (enable -f).
#
# Bash C headers:
#   Debian/Ubuntu : sudo apt-get install bash-builtins
#   Fedora/RHEL   : sudo dnf install bash-devel
#
# kv_trie also needs libexpanse (a drop-in libjudy replacement):
#   Debian/Ubuntu : sudo apt-get install libexpanse1 libexpanse-dev libjudy-compat
#   Fedora/RHEL   : sudo dnf install libexpanse libexpanse-devel libjudy-compat
#
#   make            # build fast_kv (self-contained)
#   make kv_trie    # build kv_trie (needs libexpanse)
#   make clean

BASH_INC ?= /usr/include/bash
CFLAGS   := -fPIC -shared -O2 \
            -I$(BASH_INC) -I$(BASH_INC)/include -I$(BASH_INC)/builtins

libfast_kv.so: fast_kv.c
	$(CC) $(CFLAGS) -o $@ $<

libkv_trie.so: kv_trie.c
	$(CC) $(CFLAGS) -o $@ $< -lexpanse

.PHONY: kv_trie clean
kv_trie: libkv_trie.so

clean:
	rm -f libfast_kv.so libkv_trie.so