Rubikon IB test

gistfile1.txt

[test_name]
test
[serving_properties]
engine=MPI
option.model_id=tiiuae/falcon-40b
option.task=text-generation
option.trust_remote_code=true
option.tensor_parallel_degree=8
option.max_rolling_batch_size=32
option.rolling_batch=lmi-dist
option.paged_attention=true
option.dtype=fp16
[aws_curl]
TOKENIZER=tiiuae/falcon-40b awscurl -c 1 -N 10 -X POST http://127.0.0.1:8080/invocations \
  --connect-timeout 60 \
  -H "Content-type: application/json" \
  -d '{"inputs":"The new movie that got Oscar this year","parameters":{"max_new_tokens":256, "do_sample":true}}' \
  -t -o /tmp/output.txt