ianscrivener

#Download from https://unofficial-builds.nodejs.org/download/release/ the appropriate build for armv6l, example https://unofficial-builds.nodejs.org/download/release/v18.9.1/node-v18.9.1-linux-armv6l.tar.gz
wget https://unofficial-builds.nodejs.org/download/release/v18.9.1/node-v18.9.1-linux-armv6l.tar.gz
tar -xzf node-v18.9.1-linux-armv6l.tar.gz
cd node-v18.9.1-linux-armv6l
sudo cp -R * /usr/local
node -v

ianscrivener

export AZ_MAIN_NAME=Kube2

export AZ_RG=RG_$AZ_MAIN_NAME
export AZ_VNET=VNET_$AZ_MAIN_NAME
export AZ_IP=Public_IP_$AZ_MAIN_NAME
export AZ_SUBNET=Subnet_$AZ_MAIN_NAME
export AZ_NSG=NetworkSecurityGroup_$AZ_MAIN_NAME
export AZ_NAME=VM_$AZ_MAIN_NAME
export AZ_NIC=NIC_$AZ_MAIN_NAME
ENV | grep AZ

ianscrivener

sudo systemctl stop [servicename]
sudo systemctl disable [servicename]
#rm /etc/systemd/system/[servicename]
#rm /etc/systemd/system/[servicename] symlinks that might be related
sudo systemctl daemon-reload
sudo systemctl reset-failed

ianscrivener

from time import sleep
import ssl
import json
import os
from paho.mqtt.client import Client 

username = "your VRM email"
password = "your VRM pasword"
portal_id = "your VRM portal ID"

ianscrivener

{
  "name": "My New Config Preset",
  "load_params": {
    "n_ctx": 1500,
    "n_batch": 512,
    "rope_freq_base": 10000,
    "rope_freq_scale": 1,
    "n_gpu_layers": 1,
    "use_mlock": true,
    "main_gpu": 0,

ianscrivener

# BTW: we are running in a nvidia/cuda:11.x.x-devel-ubuntu22.04

# install some extra Ubuntu packages
  apt install unzip libopenblas-dev nano git-lfs aria2c jq build-essential python3 python3-pip git -y
  pip install --upgrade pip setuptools wheel

# clone llama.cpp repo
	cd /workspace
	git clone https://github.com/ggerganov/llama.cpp.git

ianscrivener

Test

• try installing llama-cpp-python and llama-cpp-python[server] from pip ... WITH ggml-metal.metal (WITH Metal GPU support) file in python executable directory

Environment

• from previous test

Result

• llama-cpp-python[server] FAILS

Steps

ianscrivener

Test

• try rebuilding llama-cpp-python and llama-cpp-python[server] with GPU support and WITH ggml-metal.metal (WITH Metal GP support) to python executable directory

Environment

• from previous test

Result

• llama-cpp-python[server] FAILS

ianscrivener

Test

• try adding ggml-metal.metal (WITH Metal GP support) to python executable directory

Environment

• from previous test

Result

• llama-cpp-python[server] FAILS

ianscrivener

Test

• try rebuild llama.cpp with Metal GPU support

Environment

• from previous test

Result

• llama-cpp-python[server] FAILS 😞