-
-
Save donwany/b0d3c60abe1d5d8e8778ad36f6d6ddfe to your computer and use it in GitHub Desktop.
RunPod Code
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
#set you runpod key as a environment variable | |
os.environ['RUNPOD_API_KEY'] = "your_runpod_api_key" | |
import runpod | |
from IPython.display import display, Markdown | |
runpod.api_key = os.getenv("RUNPOD_API_KEY", "your_runpod_api_key") | |
if runpod.api_key == "your_runpod_api_key": | |
display( | |
Markdown( | |
"It appears that you don't have a RunPod API key. You can obtain one at [runpod.io](https://runpod.io?ref=s7508tca)" | |
) | |
) | |
raise AssertionError("Missing RunPod API key") | |
#show all possible available GPUs | |
runpod.get_gpus() | |
# Create your pod, you can set the data_center_id (optional) | |
# Decide which model you want to use, here we use falcon-40b | |
pod = runpod.create_pod( | |
name="Falcon-40B", | |
image_name="ghcr.io/huggingface/text-generation-inference:0.8", | |
gpu_type_id="NVIDIA A100 80GB PCIe", | |
cloud_type="SECURE", | |
# data_center_id="US-KS-1", | |
docker_args=f"--model-id tiiuae/falcon-40b --num-shard {gpu_count}", | |
gpu_count=2, | |
volume_in_gb=195, | |
container_disk_in_gb=5, | |
ports="80/http,29500/http", | |
volume_mount_path="/data", | |
) | |
#Create inference server using your pod LLM throw langchain | |
from langchain.llms import HuggingFaceTextGenInference | |
inference_server_url = f'https://{pod["id"]}-80.proxy.runpod.net' | |
llm = HuggingFaceTextGenInference( | |
inference_server_url=inference_server_url, | |
max_new_tokens=100, | |
top_k=10, | |
top_p=0.95, | |
typical_p=0.95, | |
temperature=0.001, | |
repetition_penalty=1.03, | |
) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment