llama_2_end_point.py

import autogen

# import autogen
autogen.oai.ChatCompletion.start_logging()
config_list = [
        {
            'model': 'meta-llama/Llama-2-70b-chat-hf',
            'api_key': 'sk-111111111111111111111111111111111111111111111111',
            'api_type': 'openai',
            'api_base': 'https://<YOUR-ENDPOINT>-8000.proxy.runpod.net/v1',
            'api_version': 'Tutorial'
        }
]
llm_config = {"config_list": config_list, "seed":44}


# Perform Completion
question = 'Who are you?'
response = autogen.oai.Completion.create(config_list=config_list, prompt=question, temperature=0, max_tokens=1000)
ans = autogen.oai.Completion.extract_text(response)[0]

print("Model response:", ans)