ref: https://nikkie-ftnext.hatenablog.com/entry/setup-phi-3-mini-ollama-llama-cpp-pythom

phi3_transformers_cpu.py

# /// script
# dependencies = ["transformers[torch]"]
# ///

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline

# 量子化した gguf モデルはconfig.jsonがなく、読み込めなかった
model_name = "microsoft/Phi-3-mini-4k-instruct"
torch.random.manual_seed(0)

model = AutoModelForCausalLM.from_pretrained(
    model_name, torch_dtype=torch.float16, trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

pipe = pipeline(
    "text-generation", model=model, tokenizer=tokenizer, device="mps"
)
prompt = "How to explain Internet to a medieval knight?"
generation_args = {
    "max_new_tokens": 256,
    "return_full_text": False,
    "temperature": 0.0,
}
output = pipe([{"role": "user", "content": prompt}], **generation_args)
print(output[0]["generated_text"])