agyaatcoder

import os
import subprocess


from modal import Image, Secret, Stub, enter, gpu, method, web_server

MODEL_DIR = "/model"
BASE_MODEL = "mistralai/Mixtral-8x7B-Instruct-v0.1"

agyaatcoder

#Meta-Llama-3-8B-Instruct is gated model and requires access on hf first to be able to successfully run this
import os
import subprocess
from modal import Image, Secret, Stub, gpu, web_server


MODEL_DIR = "/model"
MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
DOCKER_IMAGE = "ghcr.io/huggingface/text-generation-inference:1.4"
PORT = 8000

agyaatcoder

import subprocess
import os
from pathlib import Path
import socket

from modal import Image, Mount, Stub, Secret, web_server, gpu

MODEL_ID = "BAAI/bge-small-en-v1.5"
PORT = 8080
DOCKER_IMAGE = "ghcr.io/huggingface/text-embeddings-inference:86-0.4.0"

agyaatcoder

import os
import subprocess

from modal import Image, Secret, Stub, enter, gpu, method, web_server

# Constants for the model and deployment setup.
MODEL_DIR = "/model"
MODEL_ID = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"
QUANTIZATION = "awq"
DOCKER_IMAGE = "ghcr.io/huggingface/text-generation-inference:1.4"