This quanaco model for LangChain integration. Subscribe me back =)

hftg_demo_guanaco-33b-merged_qlora.py

import os

import dotenv
from langchain import HuggingFaceTextGenInference

dotenv.load_dotenv()

os.environ[
    "HUGGINGFACEHUB_API_TOKEN"
] = os.getenv("HF_API_TOKEN")

llm = HuggingFaceTextGenInference(
    inference_server_url="https://api-inference.huggingface.co/models/timdettmers/guanaco-33b-merged",
    max_new_tokens=1024,
    top_p=0.9,
    temperature=0.7,
    repetition_penalty=1.2,
)

if __name__ == "__main__":
    print(llm("How are you today?"))