near-losslessly compress text from stdin. parameter is nb of runs. using openai chat completions api.

compress-text.sh

#!/bin/bash
# Title: Compress Text
# Version: 0.1
# Date: 2024-06-12
# License: MIT
# Author: dctremblay@dasio.ca

[ $OPENAI_API_KEY ] || { echo "OPENAI_API_KEY must be set" >&2; exit 1; }

: "${SYSMSG:="You compress the following text"}"
: "${MODEL:="gpt-3.5-turbo"}"

nRuns=${1:-1}

query() {
    curl -s https://api.openai.com/v1/chat/completions \
        -H "Content-Type: application/json" \
        -H "Authorization: Bearer $OPENAI_API_KEY" \
        -d "$(jq -n --arg model "$MODEL" --argjson messages "$(cat)" '{$model,$messages}')"
}

parse() { jq -r '.choices[0].message.content'; }
toObj() { jq -n --arg role "$1" --arg content "$(cat)" '{$role,$content}'; }

toArray() { jq -s '.'; }

run() {
	(
		toObj system <<< "$SYSMSG"
		toObj user
	) | toArray | query | parse
}

end() {
	echo -e "\tbest = $bestLen chars" >&2
	echo "$bestText"
}
trap end EXIT

bestText="$(cat)"
bestLen="$(wc -m <<< "$bestText")"

echo -e "\tinput = $bestLen chars" >&2
for i in $(seq $nRuns); do
	text="$(run <<< "$bestText")"
	len="$(wc -m <<< "$text")"
	if (( len < bestLen )); then
		bestText="$text"
		bestLen="$len"
		note="  *best*"
	fi
	echo -e "\trun #$i = $len chars$note" >&2
	unset note
done