bash script to setup local llama3 model on EC2 instance using ollama and access through your machine

Llama3-on-EC2.md

#!/usr/bin/env bash

set -o errtrace
set -o nounset
set -o pipefail

# Quick steps to create the VPC/Public Subnet/EC2 in the aws account.

output=$(aws sts get-caller-identity 2>&1)
status=$?

if [ $status -ne 0 ]; then
  echo "Error: AWS credentials are missing or invalid. please set the AWS credential before executing the script"
  echo "$output"
  exit 1
fi

export AWS_REGION=us-west-1 

echo "creating VPC"
AWS_VPC=$(aws ec2 create-vpc \
--cidr-block 10.0.0.0/16 \
--query 'Vpc.{VpcId:VpcId}' \
--output text)
echo "VPC $AWS_VPC Created"

# Enable Host name
aws ec2 modify-vpc-attribute \
--vpc-id "$AWS_VPC" \
--enable-dns-hostnames "{\"Value\":true}"

# Enable DNS support
aws ec2 modify-vpc-attribute \
--vpc-id "$AWS_VPC" \
--enable-dns-support "{\"Value\":true}"


AWS_PUBLIC_SUBNET=$(aws ec2 create-subnet \
--vpc-id "$AWS_VPC" \
--cidr-block 10.0.1.0/24 \
--availability-zone us-west-1a \
--query 'Subnet.{SubnetId:SubnetId}' \
--output text)

echo "Created public Subnet with subnet ID: $AWS_PUBLIC_SUBNET"


AWS_PRIVATE_SUBNET=$(aws ec2 create-subnet \
--vpc-id "$AWS_VPC" \
--cidr-block 10.0.2.0/24 \
--availability-zone us-west-1a \
--query 'Subnet.{SubnetId:SubnetId}' \
--output text)

echo "Created private Subnet with subnet ID: $AWS_PRIVATE_SUBNET"

aws ec2 modify-subnet-attribute \
--subnet-id "$AWS_PUBLIC_SUBNET" \
--map-public-ip-on-launch

echo "Enabled auto-assign public IP on the public subnet"

AWS_INTERNET_GATEWAY=$(aws ec2 create-internet-gateway \
--query 'InternetGateway.{InternetGatewayId:InternetGatewayId}' \
--output text)

echo "created AWS Internet Gateway $AWS_INTERNET_GATEWAY"

AWS_ELASTIC_IP=$(aws ec2 allocate-address \
--domain vpc \
--query 'AllocationId' \
--output text)

echo "Allocated Elastic IP $AWS_ELASTIC_IP"

aws ec2 attach-internet-gateway \
--vpc-id "$AWS_VPC" \
--internet-gateway-id "$AWS_INTERNET_GATEWAY" \
--query 'InternetGateway.{InternetGatewayId:InternetGatewayId}' \
--output text

echo "Attached Internet GW to VPC"

AWS_ROUTE_TABLE=$(aws ec2 create-route-table \
--vpc-id "$AWS_VPC" \
--query 'RouteTable.{RouteTableId:RouteTableId}' \
--output text)

echo "Created Route Table $AWS_ROUTE_TABLE"

aws ec2 associate-route-table \
--route-table-id "$AWS_ROUTE_TABLE" \
--subnet-id "$AWS_PUBLIC_SUBNET" \
--output text

echo "Assoicated Route Table to Public Subnet"

aws ec2 create-route \
--route-table-id "$AWS_ROUTE_TABLE" \
--destination-cidr-block 0.0.0.0/0 \
--gateway-id "$AWS_INTERNET_GATEWAY" \
--output text

echo "Created Route to IGW"

AWS_SECURITY_GROUP=$(aws ec2 create-security-group \
--group-name mysshSG \
--description " Security Group to allow SSH from local machine" \
--vpc-id "$AWS_VPC" \
--query 'GroupId' \
--output text)

echo "Created SG $AWS_SECURITY_GROUP"

MY_IP=$(curl -s ifconfig.me)  
 
echo "My Machine IP $MY_IP"

aws ec2 authorize-security-group-ingress \
--group-id "$AWS_SECURITY_GROUP" \
--protocol tcp \
--port 22 \
--cidr "$MY_IP"/32 \
--output text

aws ec2 authorize-security-group-ingress \
--group-id "$AWS_SECURITY_GROUP" \
--protocol tcp \
--port 80 \
--cidr "$MY_IP"/32 \
--output text

aws ec2 authorize-security-group-ingress \
--group-id "$AWS_SECURITY_GROUP" \
--protocol tcp \
--port 443 \
--cidr "$MY_IP"/32 \
--output text

echo "Created SG rule which allow ssh from my machine IP $MY_IP/32 but it may not work due to the VPN setup, in such case may need to modify the SG rule after login into the AWS console"

AWS_AMI=$(aws ec2 describe-images \
--owners 'amazon' \
--filters 'Name=name,Values=amzn2-ami-hvm-2.0.20221004.0-x86_64-gp2' \
'Name=state,Values=available' \
--query 'sort_by(Images, &CreationDate)[-1].[ImageId]' \
--output 'text')

rm -rf ec2-krypair-file.pem
aws ec2 create-key-pair \
--key-name ec2-krypair \
--query 'KeyMaterial' \
--output text > ec2-krypair-file.pem

echo "Created key/pair pem"

chmod 400 ec2-krypair-file.pem

# here you can put all your rquired packages
cat <<EOF > install.sh
#!/bin/bash

# Update packages
sudo yum update -y
sudo curl -fsSL https://ollama.com/install.sh | sh
sudo systemctl daemon-reload
sudo systemctl restart ollama
sudo systemctl stop ollama.service
sudo cp /etc/systemd/system/ollama.service /etc/systemd/system/ollama.service.bak
sudo sed -i '/^\[Service\]/a Environment="OLLAMA_HOST=0.0.0.0"' /etc/systemd/system/ollama.service
sudo systemctl daemon-reload
sudo systemctl start ollama.service
sudo systemctl enable ollama.service
sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 11434
ollama run llama3
#sudo amazon-linux-extras install nginx1

EOF

# AMI to run the local llm with large params such as llama3:70 (with Nvidia GPU)
# AWS_AMI=ami-09b5ae78e0b580451

# Instance Types
# Instance type vCPUs Architecture Memory (GiB) Storage (GB) Storage type Network performance
# g4dn.xlarge	4	x86_64	16	125	ssd	Up to 25 Gigabit
# g4dn.2xlarge	8	x86_64	32	225	ssd	Up to 25 Gigabit
# g4dn.4xlarge	16	x86_64	64	225	ssd	Up to 25 Gigabit
# g4dn.8xlarge	32	x86_64	128	900	ssd	50 Gigabit
# g4dn.12xlarge	48	x86_64	192	900	ssd	50 Gigabit
# g4dn.16xlarge	64	x86_64	256	900	ssd	50 Gigabit
# g4dn.metal	96	x86_64	384	1800	ssd	100 Gigabit

# INSTACE_TYPE=g4dn.8xlarge
INSTACE_TYPE=t2.micro

AWS_EC2_INSTANCE=$(aws ec2 run-instances \
--image-id "$AWS_AMI" \
--instance-type "$INSTACE_TYPE" \
--key-name ec2-krypair \
--monitoring "Enabled=false" \
--security-group-ids "$AWS_SECURITY_GROUP" \
--subnet-id "$AWS_PUBLIC_SUBNET" \
--user-data file://install.sh \
--private-ip-address 10.0.1.10 \
--query 'Instances[0].InstanceId' \
--output text)

aws ec2 describe-instances \
--instance-ids "$AWS_EC2_INSTANCE"\
--query 'Reservations[*].Instances[*].[InstanceId,State.Name,PublicIpAddress]' \
--output text

AWS_PUBLIC_IP=$(aws ec2 describe-instances \
--instance-ids "$AWS_EC2_INSTANCE" \
--query 'Reservations[*].Instances[*].[PublicIpAddress]' \
--output text)

echo "Public IP $AWS_PUBLIC_IP"


AWS_PUBLIC_DNS=$(aws ec2 describe-instances \
--instance-ids "$AWS_EC2_INSTANCE" \
--query 'Reservations[*].Instances[*].[PublicDnsName]' \
--output text)

echo "Public DNS $AWS_PUBLIC_DNS"
# clean up
rm -rf install.sh
echo "ssh into the host using command ssh -i ec2-krypair-file.pem     ec2-user@$AWS_PUBLIC_DNS"
#ssh -i ec2-krypair-file.pem     ec2-user@$AWS_PUBLIC_DNS 
echo "wait for ec2 instance to come up and install the ollama, pull the model etc as defined in the user-data script"

echo "Run the following docker command to spin open-webui on local machine to connect to you LLM running on EC2 instance"
echo "###############################################################################################################################################"
echo "docker run  -p 3000:8080 -e OLLAMA_BASE_URL="http://ec2-user@$AWS_PUBLIC_DNS:80" -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main"
echo "###############################################################################################################################################"