A bash script that will download and run quantized version of mistral llm for testing

mixtral_llm_playground.sh

#!/bin/bash

# Install Python and create a virtual environment
sudo apt-get update -y
sudo apt-get install -y python3.11 python3.11-venv

# Create and activate virtual environment
python3.11 -m venv venv
source venv/bin/activate

# Install required packages
sudo apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common

# Add Docker's official GPG key
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -

# Add Docker repository to APT sources
echo "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# Add Docker repository to APT sources without user interaction
echo "Setting up Docker repository..."
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

echo \
  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# Install Docker and Docker Compose
sudo apt-get update -y
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose

# Start Docker service
sudo systemctl start docker

# Enable Docker to start on boot
sudo systemctl enable docker

# Install Hugging Face Hub
pip3 install --upgrade huggingface_hub

# Download model from Hugging Face
pip3 install huggingface_hub
huggingface-cli download TheBloke/Mistral-7B-Instruct-v0.2-GGUF mistral-7b-instruct-v0.2.Q4_K_M.gguf --local-dir . --local-dir-use-symlinks False 

# Create Docker Compose file
cat <<EOF > docker-compose.yml
version: '3'
services:
  llama:
    image: ghcr.io/ggerganov/llama.cpp:server
    ports:
      - 8080:8080
    volumes:
      - ./mistral-7b-instruct-v0.2.Q4_K_M.gguf:/models/mistral-7b-instruct-v0.2.Q4_K_M.gguf
    command: >
      -m models/mistral-7b-instruct-v0.2.Q4_K_M.gguf
      -c 512
      --host 0.0.0.0
      --port 8080
EOF

# Start Docker Compose
sudo docker-compose up -d