better.py

from transformers import AutoConfig
import time
import textwrap

from optimum.bettertransformer import BetterTransformer

from instruct_pipeline import InstructionTextGenerationPipeline
from transformers import AutoModelForCausalLM, AutoTokenizer

hf_model = "databricks/dolly-v2-2-8b"

print("loading tokenizer")
tokenizer = AutoTokenizer.from_pretrained(hf_model, padding_side="left")

print("loading model")
model = AutoModelForCausalLM.from_pretrained(hf_model, device_map="auto")
print("Model device:", model.device)

print("transforming model")
model = BetterTransformer.transform(model, keep_original_model=False).to("cuda")
print("done")
print("Model device:", model.device)
generate_text = InstructionTextGenerationPipeline(model=model, tokenizer=tokenizer)

prompts = [
"What is the process of photosynthesis in plants?",
"How does the water cycle work in the Earth's ecosystem?",
"Can you describe the basic structure of an atom?",
"What are the primary colors of light and pigment?",
"What causes the different phases of the Moon?",
"Recite the Bible, Genesis Chapter 3, verse by verse, providing commentary and analysis for each verse",
"What is the significance of the speed of light in physics?",
"How do tectonic plates interact and cause earthquakes and volcanic eruptions?",
"What is the difference between weather and climate?",
"Can you explain the greenhouse effect and its impact on global warming?",
"What are the main functions of the circulatory system in the human body?",
"What are the three states of matter and their basic characteristics?",
"What is the significance of the Gregorian calendar and how is it structured?",
"How does the digestive system process food in the human body?",
"What is the difference between renewable and non-renewable energy resources?",
"Can you name the three branches of the United States government and their functions?",
"What are the seven continents of the Earth?",
"How do magnets work and what are their basic properties?",
"What are the major layers of Earth's atmosphere and their characteristics?",
"What is the process of natural selection and how does it relate to evolution?",
"What are the four fundamental forces of nature?",
"How does the human immune system work to protect the body from disease?",
"Can you explain the basic principles of supply and demand in economics?",
"What are the three major types of rocks and how do they form?",
"How do electric circuits work and what are their basic components?",
"What are the primary differences between plant and animal cells?"
]
for prompt in prompts:
    print("Q.", prompt)
    start_time = time.time()
    output = generate_text(prompt)
    end_time = time.time()
    print("A.", textwrap.fill(output, 80))
    elapsed_time = end_time - start_time
    print(f"[Words/sec: {len(output)/elapsed_time:.2f}]")
    print("===")