Rubikon IB Mistral 7B

gistfile1.txt

[test_name]
concurrency1
[serving_properties]
engine=Python
option.tensor_parallel_degree=1
option.rolling_batch=vllm
option.model_id=mistralai/Mistral-7B-v0.1
option.max_rolling_batch_size=32
[aws_curl]
TOKENIZER=mistralai/Mistral-7B-v0.1 ./awscurl -c 1 -N 10 \
-X POST http://127.0.0.1:8080/invocations   \
--connect-timeout 60   -H "Content-type: application/json"   \
-d '{"inputs":"The new movie that got Oscar this year","parameters":{"max_new_tokens":256, "do_sample":true}}'   \
-t -o /tmp/output.txt
[test_name]
concurrency16
[serving_properties]
engine=Python
option.tensor_parallel_degree=1
option.rolling_batch=vllm
option.model_id=mistralai/Mistral-7B-v0.1
option.max_rolling_batch_size=32
[aws_curl]
TOKENIZER=mistralai/Mistral-7B-v0.1 ./awscurl -c 16 -N 10 \
-X POST http://127.0.0.1:8080/invocations   \
--connect-timeout 60   -H "Content-type: application/json"   \
-d '{"inputs":"The new movie that got Oscar this year","parameters":{"max_new_tokens":256, "do_sample":true}}'   \
-t -o /tmp/output.txt
[test_name]
concurrency32
[serving_properties]
engine=Python
option.tensor_parallel_degree=1
option.rolling_batch=vllm
option.model_id=mistralai/Mistral-7B-v0.1
option.max_rolling_batch_size=32
[aws_curl]
TOKENIZER=mistralai/Mistral-7B-v0.1 ./awscurl -c 32 -N 10 \
-X POST http://127.0.0.1:8080/invocations   \
--connect-timeout 60   -H "Content-type: application/json"   \
-d '{"inputs":"The new movie that got Oscar this year","parameters":{"max_new_tokens":256, "do_sample":true}}'   \
-t -o /tmp/output.txt