IB TGI

gistfile1.txt

[test_name]
mistral
[container]
ghcr.io/huggingface/text-generation-inference:1.3.4
[vars]
CONCURRENCY={1,16,32}
[env]
MODEL_ID=mistralai/Mistral-7B-v0.1
NUM_SHARD=1
[aws_curl]
TOKENIZER=mistralai/Mistral-7B-v0.1 ./awscurl -c $CONCURRENCY -N 10 \
-X POST http://127.0.0.1:8080/generate   \
--connect-timeout 60   -H "Content-type: application/json"   \
-d '{"inputs":"The new movie that got Oscar this year","parameters":{"max_new_tokens":256, "do_sample":true}}'   \
-t -o /tmp/output.txt