Simple program that uses GNU Parallel to make parllel curl requests

parallel_curl.sh

#!/bin/bash

# $1 -> input file: each line is a prompt to model.
# $2 -> no. of threads to use
# usage: time ./script.sh data/test_data.ldjson 128 > result.ldjson

# File containing payloads
FILE=$1 #"data/test_data.ldjson"

# Number of parallel requests
PARALLEL_REQUESTS=$2

# Function to make curl request
make_curl_request() {
  local payload="$1"
  ##### TGI payload format
  curl -XPOST "0.0.0.0:9091/generate" -H "Content-Type: application/json" -d "{\"inputs\": $payload, \"parameters\": {\"max_new_tokens\": 400,\"do_sample\":  false,\"best_of\": null,\"repet
ition_penalty\": 1,\"return_full_text\": false,\"seed\": null,\"stop_sequences\": null,\"temperature\": 0.1,\"top_k\": 100,\"top_p\": 0.3,\"truncate\": null,\"typical_p\": null,\"watermark\"
: false,  \"decoder_input_details\": false}}"
  echo ""
}

export -f make_curl_request

#head -n 10000 "$FILE"| jq '.text' | parallel -j 10 make_curl_request

# Use parallel to process lines in parallel
cat "$FILE" | parallel -j "$PARALLEL_REQUESTS" make_curl_request