-
-
Save SlyEcho/7ab8914dfd5d75483cbc2bb94308755a to your computer and use it in GitHub Desktop.
EstGPT script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
MASTODON_URL="https://est.social" | |
MASTODON_TOKEN='xxx' | |
DEEPL_TOKEN='xxx' | |
LLAMA_PATH="$HOME/src/llama.cpp/build/bin/main" | |
LLAMA_MODEL="$HOME/src/llama.cpp/models/alpaca-native-13b-q4_0.bin" | |
LLAMA_ARGS="--keep 21 -n -1 -t 8 -b 4 -c 2048" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -eu | |
set -o pipefail | |
source .env | |
if ! [ -z "${1:-}" ]; then | |
LAST_ID="$1" | |
curl \ | |
-o notification.json \ | |
--silent \ | |
--request GET \ | |
--url "${MASTODON_URL}/api/v1/notifications/${LAST_ID}" \ | |
--header "Authorization: Bearer ${MASTODON_TOKEN}" | |
jq '[.]' notification.json > notification_.json | |
mv -f notification_.json notification.json | |
else | |
if [ -s last_mention.json ]; then | |
LAST_ID=$(jq -r .id last_mention.json) | |
fi | |
curl \ | |
-o notification.json \ | |
--silent \ | |
--request GET \ | |
--url "${MASTODON_URL}/api/v1/notifications?types%5B%5D=mention&min_id=${LAST_ID:-}&limit=1" \ | |
--header "Authorization: Bearer ${MASTODON_TOKEN}" | |
fi | |
COUNT=$(jq -r '.|length' notification.json) | |
if [[ $COUNT = 0 ]]; then | |
echo "No mentions :(" >&2 | |
exit 0 | |
fi | |
eval $(jq -r ' | |
.[0] as $note | [ | |
"REPLYTO_STATUS_ID=\($note.status.in_reply_to_id | @sh)", | |
"LAST_ID=\($note.id | @sh)", | |
"STATUS_ID=\($note.status.id | @sh)", | |
"USERNAME=\($note.account.acct | @sh)", | |
"VISIBILITY=\($note.status.visibility | @sh)", | |
"MENTIONS=\([$note.status.mentions[]|"@\(.acct)","@\($note.account.acct)"]|unique|join(" ")|gsub("@bot";"")|@sh)" | |
] | join("\n") | |
' notification.json) | |
STATUS_SOURCE=$(jq -r ' | |
.[0].status.content | |
| gsub("</p>";"\n") | |
| gsub("<br\\s*/?>";"\n") | |
| gsub("<[^>]*>";"") | |
| gsub("@\\w+";"") | |
| gsub(""";"\"") | |
| gsub("'";"\u0027") | |
| gsub("^\\s+|\\s+$";"") | |
' notification.json) | |
echo "COUNT = $COUNT" | |
echo "LAST_ID = $LAST_ID" | |
echo "STATUS_ID = $STATUS_ID" | |
echo "REPLYTO_STATUS_ID = $REPLYTO_STATUS_ID" | |
echo "USERNAME = $USERNAME" | |
echo "MENTIONS = $MENTIONS" | |
echo "VISIBILITY = $VISIBILITY" | |
echo "STATUS_SOURCE = ${STATUS_SOURCE}" | |
echo "${STATUS_SOURCE}" | curl \ | |
-o translation.json \ | |
--silent \ | |
--request POST \ | |
--url https://api-free.deepl.com/v2/translate \ | |
--header "Authorization: DeepL-Auth-Key ${DEEPL_TOKEN}" \ | |
--header 'Content-Type: application/x-www-form-urlencoded' \ | |
--data-urlencode "text@-" \ | |
--data target_lang=en | |
TRANSLATED_SOURCE=$(jq -r '.translations[0].text|gsub("^\\s+|\\s+$";"")' translation.json) | |
DETECTED_LANG=$(jq -r '.translations[0].detected_source_language|ascii_downcase' translation.json) | |
echo "TRANSLATED_SOURCE = ${TRANSLATED_SOURCE}" | |
echo "DETECTED_LANG = ${DETECTED_LANG}" | |
mkdir -p prompts | |
if [ "${REPLYTO_STATUS_ID:-null}" != "null" ] && [ -s "prompts/${REPLYTO_STATUS_ID}.txt" ]; then | |
jq -sRr 'split("### Instruction:")|.[0] as $ins|.[1:]|[$ins,.[-3:][]]|join("### Instruction:")' "prompts/${REPLYTO_STATUS_ID}.txt" > "prompt.txt" | |
else | |
echo -n 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' > "prompt.txt" | |
fi | |
printf '\n\n### Instruction:\n%s\n\n### Response: \n' "${TRANSLATED_SOURCE}" >> "prompt.txt" | |
"${LLAMA_PATH}" -m "${LLAMA_MODEL}" ${LLAMA_ARGS} -f "prompt.txt" 2>> llama_error.log | tee llama_result.txt | |
RESULT_TEXT=$(jq -s -R -r '.|split("### Response:")|.[-1]|split("### Instruction:")|.[0]|gsub("^\\s+|\\s+$";"")' llama_result.txt) | |
echo "RESULT_TEXT=${RESULT_TEXT}" | |
echo "${RESULT_TEXT}" >> "prompt.txt" | |
if [ "${DETECTED_LANG}" != "en" ]; then | |
echo "${RESULT_TEXT}" | curl \ | |
-o translation.json \ | |
--silent \ | |
--request POST \ | |
--url https://api-free.deepl.com/v2/translate \ | |
--header "Authorization: DeepL-Auth-Key ${DEEPL_TOKEN}" \ | |
--header 'Content-Type: application/x-www-form-urlencoded' \ | |
--data-urlencode "text@-" \ | |
--data target_lang=${DETECTED_LANG} | |
REPLY_TEXT=$(jq -r '.translations[0].text' translation.json) | |
else | |
REPLY_TEXT="${RESULT_TEXT}" | |
fi | |
echo "REPLY_TEXT=${REPLY_TEXT}" | |
if [ "${VISIBILITY}" == "direct" ]; then | |
printf "%s\n\n%s" "${MENTIONS}" "${REPLY_TEXT}" > post_text.txt | |
else | |
printf "%s ❰%s❱\n\n➥ %s" "${MENTIONS}" "${STATUS_SOURCE}" "${REPLY_TEXT}" > post_text.txt | |
fi | |
curl \ | |
-o reply-post.json \ | |
--silent \ | |
--request POST \ | |
--url "${MASTODON_URL}/api/v1/statuses" \ | |
--header "Authorization: Bearer ${MASTODON_TOKEN}" \ | |
--header 'Content-Type: application/x-www-form-urlencoded' \ | |
--data-urlencode "Idempotency-Key=REPLYTO-${STATUS_ID}" \ | |
--data-urlencode 'status@post_text.txt' \ | |
--data "in_reply_to_id=${STATUS_ID}" \ | |
--data "visibility=${VISIBILITY}" \ | |
--data "language=${DETECTED_LANG}" | |
eval $(jq -r '[ | |
"REPLY_URL=\(.url | @sh)", | |
"REPLY_ID=\(.id | @sh)" | |
]|join("\n")' reply-post.json) | |
echo "Reply at: ${REPLY_URL}" | |
jq -n '{id:$id}' --arg id "$LAST_ID" > last_mention.json | |
mv -v prompt.txt "prompts/${REPLY_ID}.txt" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
EstGPT
Basically a dumb wrapper around llama.cpp. This basically is only workable now after @jart's
mmap()
changes to load the model instantly.There are a lot of LLaMa models out there, but this one is made to work with Alpaca with the way the prompts are set up.
Conversations
The script takes the first mention after an id that it remembers in
last_mention.json
, or a notification ID can be passed on the command line for resending replies.The user's comments are translated with DeepL to English, because that language works best with the models. Then it generates a response using llama.cpp and translates back to the original language (if it wasn't in English).
It posts a reply and then dumps the llama.cpp prompt to a file with the ID of the new reply. The idea is that when the user asks another question in a row, the bot can reload the past prompts as a "memory" because the new comment is linked to the previous ID. It does have some limits how much tokens (text) the model can eat, because of the way transformer models work, so it cuts out anything but the last 3 questions-answers.
In non-direct messages it also shows the original question, that was a feature request.
Code
Don't look at it too hard. Mostly hacky bash and jq scripts.