How to use databricks/dolly-v2-12b tutorial

0 tutorial.txt

databricks/dolly-v2-12b model : https://huggingface.co/databricks/dolly-v2-12b

python 3.10.6 download link : https://www.python.org/ftp/python/3.10.6/python-3.10.6-amd64.exe

install it into your C drive directly better - make sure to add path

git download link : https://git-scm.com/downloads

git large download link : https://git-lfs.com/

after installing git open cmd and type this command : git lfs install

First move into the folder where you want to generate your virtual environment folder

Then type open cmd and type this command : python -m venv vendolly

download and install cuda : https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64

pip install accelerate
# pip install bitsandbytes // this doesnt work for windows
pip install transformers[torch]==4.25.1
pip install git+https://github.com/Keith-Hon/bitsandbytes-windows.git // this one is for windows
pip install gradio

starting source of the shared gradio (improved by me) : https://huggingface.co/spaces/RamAnanth1/Dolly-v2

1 dolly.py

import gradio as gr
from transformers import *
import torch

theme = gr.themes.Monochrome(
    primary_hue="indigo",
    secondary_hue="blue",
    neutral_hue="slate",
    radius_size=gr.themes.sizes.radius_sm,
    font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
)

#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-12b", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-12b", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto",task="text-generation")
#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-12b", torch_dtype=torch.float16, trust_remote_code=True, device_map="auto",task="text-generation")
#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-12b", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto",task="text-generation", max_new_tokens=128)
#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-7b", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto",task="text-generation")
#instruct_pipeline = pipeline(model="F:/Dolly 2.0/dolly-v2-7b", torch_dtype=torch.bfloat16, load_in_8bit=True, trust_remote_code=True, device_map="auto",task="text-generation",max_new_tokens=2048)


model_name = "F:/Dolly 2.0/dolly-v2-7b"
tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
#model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True, trust_remote_code=True)

end_key_token_id = tokenizer.encode("### End")[0]

instruct_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer,pad_token_id=tokenizer.pad_token_id, eos_token_id=end_key_token_id)

def generate(instruction): 
    input_ids = tokenizer.encode(instruction, return_tensors="pt")
    input_ids = input_ids.to(model.device)  # Move input_ids to the same device as the model
    generated_output = model.generate(input_ids, max_length=256,pad_token_id=tokenizer.pad_token_id, eos_token_id=end_key_token_id)
    dd = tokenizer.decode(generated_output[0])
    return dd


examples = [
    "Instead of making a peanut butter and jelly sandwich, what else could I combine peanut butter with in a sandwich? Give five ideas",
    "How do I make a campfire?",
    "Write me a tweet about the release of Dolly 2.0, a new LLM"  
]


def process_example(args):
    for x in generate(args):
        pass
    return x

css = ".generating {visibility: hidden}"

with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
    with gr.Column():
        gr.Markdown(
            """ ## Dolly 2.0
            Dolly 2.0 is a 12B parameter language model based on the EleutherAI pythia model family and fine-tuned exclusively on a new, high-quality human generated instruction following dataset, crowdsourced among Databricks employees. For more details, please refer to the [model card](https://huggingface.co/databricks/dolly-v2-12b)
            
            Type in the box below and click the button to generate answers to your most pressing questions!
            
      """
        )
        gr.HTML("<p>Check out SECourses for AI, Stable Diffusion, ML and Programming Related Full Free Courses, Tutorials and Guides  : <a target='_blank' style='display:inline-block' href='https://www.youtube.com/@SECourses' alt='https://www.youtube.com/@SECourses'>https://www.youtube.com/@SECourses</a> </p>")

        with gr.Row():
            with gr.Column(scale=3):
                instruction = gr.Textbox(placeholder="Enter your question here", label="Question", elem_id="q-input")

                with gr.Box():
                    gr.Markdown("**Answer**")
                    output = gr.Markdown(elem_id="q-output")
                submit = gr.Button("Generate", variant="primary")
                gr.Examples(
                    examples=examples,
                    inputs=[instruction],
                    cache_examples=False,
                    fn=process_example,
                    outputs=[output],
                )


    submit.click(generate, inputs=[instruction], outputs=[output])
    instruction.submit(generate, inputs=[instruction], outputs=[output])

demo.queue(concurrency_count=1).launch(debug=True)

How can install the dolly model on Google Colab pro? also, How can we deploy web version?

How can install the dolly model on Google Colab pro? also, How can we deploy web version?

for web version you can start with --share

so it will be

demo.queue(concurrency_count=1).launch(debug=True,share=True)

even on colab this will give you public link to use web interface

colab this will give you public link to use web interface

How colab will give a public link to use the web interface?