vllm.py

from openai import OpenAI

openai_api_key = "EMPTY"
# openai_api_base = "http://localhost:8000/v1"
openai_api_base = "https://jarvisai-vllm.apps.int.stc.ai.dev.us-east-1.aws.paas.redhat.com/v1"
model = "mistralai/Mistral-7B-Instruct-v0.2"
prompt = "You are an expert programmer that writes simple, concise code and explanations. Write a python function to generate the nth fibonacci number."

client = OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)

chat_completion = client.chat.completions.create(
    messages=[
        {
            "role": "user",
            "content": prompt,
        }
    ],
    model=model,
    stream=True
)

for x in chat_completion:
    print(x.choices[0].delta.content, end="")
print("Chat response:", chat_completion)