abetlen

from llama_cpp import Llama
from llama_cpp.llama_chat_format import Llava15ChatHandler
chat_handler = Llava15ChatHandler(clip_model_path="llava-1.6-mistral-7b-gguf/mmproj-model-f16.gguf")
llm = Llama(
  model_path="llava-1.6-mistral-7b-gguf/llava-v1.6-mistral-7b.Q4_K_M.gguf",
  chat_handler=chat_handler,
  n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
  logits_all=True,# needed to make llava work
  n_gpu_layers=-1
)