autogen_vllm.py

from autogen import OpenAIWrapper
import json

config_list = [
    {
        "model": model_path,
        "api_key": 'sk-111111111111111111111111111111111111111111111111',
        "base_url": vllm_endpoint, # Replace with your Runpod endpoint. Ensure that you tack "/v1" on the end of your URL.
    },
]

# Now pass this list when you create an instance of OpenAIWrapper
client = OpenAIWrapper(config_list=config_list)

def valid_json_filter(response, **_):
    for text in OpenAIWrapper.extract_text_or_completion_object(response):
        try:
            json.loads(text)
            return True
        except ValueError:
            pass
    return False


prompt = "What is AI?"

response = client.create(
    prompt=prompt,
    filter_func=valid_json_filter,
    max_tokens=250
)

response.choices[0].text