EstGPT script

.env

MASTODON_URL="https://est.social"
MASTODON_TOKEN='xxx'
DEEPL_TOKEN='xxx'
LLAMA_PATH="$HOME/src/llama.cpp/build/bin/main"
LLAMA_MODEL="$HOME/src/llama.cpp/models/alpaca-native-13b-q4_0.bin"
LLAMA_ARGS="--keep 21 -n -1 -t 8 -b 4 -c 2048"

replybot.sh

#!/bin/bash

set -eu
set -o pipefail

source .env

if ! [ -z "${1:-}" ]; then

    LAST_ID="$1"

    curl \
        -o notification.json \
        --silent \
        --request GET \
        --url "${MASTODON_URL}/api/v1/notifications/${LAST_ID}" \
        --header "Authorization: Bearer ${MASTODON_TOKEN}"

    jq '[.]' notification.json > notification_.json
    mv -f notification_.json notification.json

else

    if [ -s last_mention.json ]; then
        LAST_ID=$(jq -r .id last_mention.json)
    fi

    curl \
        -o notification.json \
        --silent \
        --request GET \
        --url "${MASTODON_URL}/api/v1/notifications?types%5B%5D=mention&min_id=${LAST_ID:-}&limit=1" \
        --header "Authorization: Bearer ${MASTODON_TOKEN}"

fi

COUNT=$(jq -r '.|length' notification.json)

if [[ $COUNT = 0 ]]; then
    echo "No mentions :(" >&2
    exit 0
fi

eval $(jq -r '
    .[0] as $note | [
        "REPLYTO_STATUS_ID=\($note.status.in_reply_to_id | @sh)",
        "LAST_ID=\($note.id | @sh)",
        "STATUS_ID=\($note.status.id | @sh)",
        "USERNAME=\($note.account.acct | @sh)",
        "VISIBILITY=\($note.status.visibility | @sh)",
        "MENTIONS=\([$note.status.mentions[]|"@\(.acct)","@\($note.account.acct)"]|unique|join(" ")|gsub("@bot";"")|@sh)"
    ] | join("\n")
' notification.json)

STATUS_SOURCE=$(jq -r '
    .[0].status.content
        | gsub("</p>";"\n")
        | gsub("<br\\s*/?>";"\n")
        | gsub("<[^>]*>";"")
        | gsub("@\\w+";"")
        | gsub("&quot;";"\"")
        | gsub("&#39;";"\u0027")
        | gsub("^\\s+|\\s+$";"")
' notification.json)

echo "COUNT = $COUNT"
echo "LAST_ID = $LAST_ID"
echo "STATUS_ID = $STATUS_ID"
echo "REPLYTO_STATUS_ID = $REPLYTO_STATUS_ID"
echo "USERNAME = $USERNAME"
echo "MENTIONS = $MENTIONS"
echo "VISIBILITY = $VISIBILITY"
echo "STATUS_SOURCE = ${STATUS_SOURCE}"

echo "${STATUS_SOURCE}" | curl \
    -o translation.json \
    --silent \
    --request POST \
    --url https://api-free.deepl.com/v2/translate \
    --header "Authorization: DeepL-Auth-Key ${DEEPL_TOKEN}" \
    --header 'Content-Type: application/x-www-form-urlencoded' \
    --data-urlencode "text@-" \
    --data target_lang=en

TRANSLATED_SOURCE=$(jq -r '.translations[0].text|gsub("^\\s+|\\s+$";"")' translation.json)
DETECTED_LANG=$(jq -r '.translations[0].detected_source_language|ascii_downcase' translation.json)


echo "TRANSLATED_SOURCE = ${TRANSLATED_SOURCE}"
echo "DETECTED_LANG = ${DETECTED_LANG}"

mkdir -p prompts

if [ "${REPLYTO_STATUS_ID:-null}" != "null" ] && [ -s "prompts/${REPLYTO_STATUS_ID}.txt" ]; then
    jq -sRr 'split("### Instruction:")|.[0] as $ins|.[1:]|[$ins,.[-3:][]]|join("### Instruction:")' "prompts/${REPLYTO_STATUS_ID}.txt" > "prompt.txt"
else
    echo -n 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' > "prompt.txt"
fi

printf '\n\n### Instruction:\n%s\n\n### Response: \n' "${TRANSLATED_SOURCE}" >> "prompt.txt"

"${LLAMA_PATH}" -m "${LLAMA_MODEL}" ${LLAMA_ARGS} -f "prompt.txt" 2>> llama_error.log | tee llama_result.txt

RESULT_TEXT=$(jq -s -R -r '.|split("### Response:")|.[-1]|split("### Instruction:")|.[0]|gsub("^\\s+|\\s+$";"")' llama_result.txt)

echo "RESULT_TEXT=${RESULT_TEXT}"

echo "${RESULT_TEXT}" >> "prompt.txt"

if [ "${DETECTED_LANG}" != "en" ]; then

    echo "${RESULT_TEXT}" | curl \
        -o translation.json \
        --silent \
        --request POST \
        --url https://api-free.deepl.com/v2/translate \
        --header "Authorization: DeepL-Auth-Key ${DEEPL_TOKEN}" \
        --header 'Content-Type: application/x-www-form-urlencoded' \
        --data-urlencode "text@-" \
        --data target_lang=${DETECTED_LANG}

    REPLY_TEXT=$(jq -r '.translations[0].text' translation.json)
else
    REPLY_TEXT="${RESULT_TEXT}"
fi

echo "REPLY_TEXT=${REPLY_TEXT}"

if [ "${VISIBILITY}" == "direct" ]; then 
    printf "%s\n\n%s" "${MENTIONS}" "${REPLY_TEXT}" > post_text.txt
else
    printf "%s ❰%s❱\n\n➥ %s" "${MENTIONS}" "${STATUS_SOURCE}" "${REPLY_TEXT}" > post_text.txt
fi

curl \
    -o reply-post.json \
    --silent \
    --request POST \
    --url "${MASTODON_URL}/api/v1/statuses" \
    --header "Authorization: Bearer ${MASTODON_TOKEN}" \
    --header 'Content-Type: application/x-www-form-urlencoded' \
    --data-urlencode "Idempotency-Key=REPLYTO-${STATUS_ID}" \
    --data-urlencode 'status@post_text.txt' \
    --data "in_reply_to_id=${STATUS_ID}" \
    --data "visibility=${VISIBILITY}" \
    --data "language=${DETECTED_LANG}"

eval $(jq -r '[
        "REPLY_URL=\(.url | @sh)",
        "REPLY_ID=\(.id | @sh)"
    ]|join("\n")' reply-post.json)

echo "Reply at: ${REPLY_URL}"

jq -n '{id:$id}' --arg id "$LAST_ID" > last_mention.json

mv -v prompt.txt "prompts/${REPLY_ID}.txt"

EstGPT

Basically a dumb wrapper around llama.cpp. This basically is only workable now after @jart's mmap() changes to load the model instantly.

There are a lot of LLaMa models out there, but this one is made to work with Alpaca with the way the prompts are set up.

Conversations

The script takes the first mention after an id that it remembers in last_mention.json, or a notification ID can be passed on the command line for resending replies.

The user's comments are translated with DeepL to English, because that language works best with the models. Then it generates a response using llama.cpp and translates back to the original language (if it wasn't in English).

It posts a reply and then dumps the llama.cpp prompt to a file with the ID of the new reply. The idea is that when the user asks another question in a row, the bot can reload the past prompts as a "memory" because the new comment is linked to the previous ID. It does have some limits how much tokens (text) the model can eat, because of the way transformer models work, so it cuts out anything but the last 3 questions-answers.

In non-direct messages it also shows the original question, that was a feature request.

Code

Don't look at it too hard. Mostly hacky bash and jq scripts.