Inkbot example shell script

chat_inkbot.sh

#!/bin/bash

set -e

cd "$(dirname "$0")/.." || exit

MODEL="${MODEL:-./models/inkbot-13b-4k.Q4_K_M.gguf}"
PROMPT_TEMPLATE=${PROMPT_TEMPLATE:-./inkbot-chat.txt}
USER_NAME="${USER_NAME:-<#user#>}"
AI_NAME="${AI_NAME:-<#bot#>}"

# Adjust to the number of CPU cores you want to use.
N_THREAD="${N_THREAD:-1}"
# Number of tokens to predict (made it larger than default because we want a long interaction)
N_PREDICTS="${N_PREDICTS:-4096}"

# Note: you can also override the generation options by specifying them on the command line:
# For example, override the context size by doing: ./chatLLaMa --ctx_size 1024
GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 4096 --temp 0.6 --top_k 30 --top_p 0.4 --repeat_last_n 4096 --batch_size 1024 --repeat_penalty 1.05}"


./main $GEN_OPTIONS \
  --model "$MODEL" \
  --threads "$N_THREAD" \
  -ngl 40 \
  --n_predict "$N_PREDICTS" \
  --color \
  --interactive \
  --multiline-input \
  --reverse-prompt "${USER_NAME}" \
  --in-prefix "${USER_NAME}
" \
  --in-suffix "${AI_NAME}
" \
  --file ${PROMPT_TEMPLATE} \
  "$@"