llm-demo.red

Red [
	Title:	"llm-demo — a statically-linked LLM inside a Red executable"
	File:	%llm-demo.red
	Note: {
		llama.cpp + ggml statically linked via the llama-red shim: one
		standalone 32-bit exe, no DLLs, running quantized GGUF models.

		Build:	redc -r -s -t Windows llm-demo.red
		Run:	llm-demo <model.gguf> [prompt words...]
		        llm-demo models/Qwen3-0.6B-Q8_0.gguf Why is the sky blue? /no_think
	}
]

#system-global [
	#include %llama-red.reds

	llm: as int-ptr! 0							;-- one model handle, R/S side
]

;-- Load a GGUF model; temp 0.0 = greedy (deterministic)
load-model: routine [
	path		[string!]
	n-ctx		[integer!]
	threads		[integer!]
	temp		[float!]
	verbose?	[logic!]
	return:		[logic!]
	/local
		len		[integer!]
		cstr	[c-string!]
][
	len: -1
	cstr: unicode/to-utf8 path :len
	unless verbose? [lr-quiet]
	lr-init
	llm: lr-load cstr n-ctx threads temp 42
	llm <> null
]

;-- Feed the prompt (chat-templated), stream the completion to stdout.
;-- Returns tokens generated, negative on error.
generate: routine [
	prompt		[string!]
	max-toks	[integer!]
	return:		[integer!]
	/local
		len		[integer!]
		cstr	[c-string!]
		buf		[byte-ptr!]
		tail	[byte-ptr!]
		n		[integer!]
		count	[integer!]
		t0		[integer!]
		t1		[integer!]
		rate	[integer!]
][
	if llm = null [return -100]
	len: -1
	cstr: unicode/to-utf8 prompt :len
	n: lr-start llm cstr 1
	if n < 0 [return -1]
	buf: allocate 512
	count: 0
	t0: lr-ticks
	n: lr-next llm buf 511
	while [all [n >= 0  count < max-toks]][
		tail: buf + n							;-- NUL-terminate the piece
		tail/1: null-byte						;-- (no paren indexing in R/S)
		print as c-string! buf					;-- stream it
		count: count + 1
		n: lr-next llm buf 511
	]
	t1: lr-ticks
	free buf
	if t1 > t0 [
		rate: count * 10000 / (t1 - t0)			;-- tok/s ×10
		print [lf "--- " count " tokens - " rate / 10 "." rate // 10 " tok/s ---" lf]
	]
	count
]

last-llm-error: routine [return: [string!]][
	string/load lr-last-error  length? lr-last-error  UTF-8
]

model-info: routine [return: [string!]][
	string/load lr-system-info  length? lr-system-info  UTF-8
]

free-model: routine [][
	if llm <> null [
		lr-free llm
		llm: as int-ptr! 0
	]
]

;=============================================================================
;	CLI
;=============================================================================

args: system/options/args
either any [none? args  empty? args][
	print "usage: llm-demo <model.gguf> [--ctx N] [prompt ...]"
][
	model-path: first args
	args: next args
	ctx-size: 4096
	threads: 0									;-- 0 = auto (shim: half the cores)
	verbose?: no
	if all [not empty? args  "--verbose" = first args][
		verbose?: yes
		args: next args
	]
	if all [1 < length? args  "--ctx" = first args][
		ctx-size: to integer! second args
		args: skip args 2
	]
	if all [1 < length? args  "--threads" = first args][
		threads: to integer! second args
		args: skip args 2
	]
	prompt: either empty? args [
		copy "Give me one surprising fact about the color red. /no_think"
	][
		form args
	]
	print ["model :" model-path]
	print ["ctx   :" ctx-size]
	print ["prompt:" prompt]
	print ""
	if verbose? [print ["sysinfo:" model-info]]
	either load-model model-path ctx-size threads 0.0 verbose? [
		if negative? generate prompt 512 [
			print ["** generation error:" last-llm-error]
		]
		free-model
	][
		print ["** load error:" last-llm-error]
	]
]