A single script to get falcon-40b-instruct running on a Lambda Labs H100

falconer.sh

#!/bin/bash
# -----
# WARNING: this does NOT work on Lambda Labs H100 instances... yet
# -----

# From: https://huggingface.co/tiiuae/falcon-40b/discussions/38#6479de427c18dca75e9a0903
pip install git+https://www.github.com/huggingface/transformers@2e2088f24b60d8817c74c32a0ac6bb1c5d39544d
pip install huggingface-hub==0.15.1
pip install tokenizers==0.13.3
pip install safetensors==0.3.1
pip install git+https://github.com/huggingface/accelerate@040f178569fbfe7ab7113af709dc5a7fa09e95bd
pip install bitsandbytes==0.39.0
pip install einops==0.6.1

# Write out a test script for a completion
python_script=$(cat <<EOF
from transformers import AutoTokenizer, AutoModelForCausalLM
import transformers
import torch
import time

start = time.time()

model = "tiiuae/falcon-40b-instruct"

print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(model)
pipeline = transformers.pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True,
    device_map="auto",
)
sequences = pipeline(
   "Describe the current Solar System in as much detail as possible.",
    max_length=200,
    do_sample=True,
    top_k=1,
    num_return_sequences=1,
    eos_token_id=tokenizer.eos_token_id,
)
for seq in sequences:
    print(f"Result: {seq['generated_text']}")


end = time.time()

print(f"Took {round(end - start, 2)} seconds")
EOF
)

# Write the Python script to a file
echo "$python_script" > run.py

# Execute the Python script
python run.py 2>&1 | tee run_out_and_errs.txt