Skip to content

Instantly share code, notes, and snippets.

@jocot
Created January 7, 2024 00:28
Show Gist options
  • Save jocot/27fd0c6b370d5afd317b2a5602f4dea4 to your computer and use it in GitHub Desktop.
Save jocot/27fd0c6b370d5afd317b2a5602f4dea4 to your computer and use it in GitHub Desktop.
ollama python utility script
#!/usr/bin/env python3
import json
import os
import shutil
import sys
OLLAMA_PATH = '/home/ollama'
def loadJSON(filename):
if os.path.exists(filename):
with open(filename) as f:
return json.load(f)
return []
def saveJSON(filename, data):
folder = os.path.dirname(filename)
os.makedirs(folder, exist_ok=True)
f = open(filename, 'w')
f.write(json.dumps(data, indent=2))
class MediaBlob:
def __init__(self, type = '', digest = '', size = 0):
self.type = type
self.digest = digest
self.size = size
def path(self):
return os.path.join("/home/ollama/models/blobs/", self.digest)
def getSize(self):
return os.path.exists(self.path()) and os.path.getsize(self.path())
def ok(self):
return os.path.exists(self.path()) and self.size == self.getSize()
@classmethod
def ParseType(self, type):
if 'container.image' in type:
return 'config'
return type.split('.')[-1]
@classmethod
def Parse(self, jdata: dict):
rv = MediaBlob()
rv.type = MediaBlob.ParseType(jdata['mediaType'])
rv.digest = jdata['digest']
rv.size = jdata['size']
return rv
def to_dict(self):
return {
'type': self.type,
'digest': self.digest,
'size': self.size,
'path': self.path(),
'actual_size': self.getSize(),
'ok': self.ok(),
'content': self.content(),
}
def content(self):
if self.type != 'model' and self.getSize() < 1024:
if self.type in ['config', 'params']:
return loadJSON(self.path())
return open(self.path(), 'r').read()
class Manifest:
def __init__(self):
self.name = ''
self.config: MediaBlob = MediaBlob()
self.layers: dict[str, MediaBlob] = {}
@classmethod
def LibPath(self):
return os.path.join(OLLAMA_PATH, 'models', 'manifests', 'registry.ollama.ai', 'library')
@classmethod
def List(self, opt = None):
path = Manifest.LibPath()
dirs = os.listdir(path)
rv = []
for dir in dirs:
filenames = os.listdir(os.path.join(path, dir))
for filename in filenames:
name = f"{dir}:{filename}"
line = name
show = True
if opt:
show = False
mf = Manifest.Load(name)
if opt == 'sha':
line += ',' + mf.layers['model'].digest
show = True
elif opt == 'ctx':
params = mf.layers['params'].content()
if 'num_ctx' in params:
line += ' ' + str(params['num_ctx'])
show = True
if show:
rv.append(line)
rv.sort()
return rv
@classmethod
def Load(self, name):
filename = self.path(name)
jdata = loadJSON(filename)
rv = Manifest()
if len(jdata) == 0:
return rv
rv.name = name
rv.config = MediaBlob.Parse(jdata['config'])
for layer in jdata['layers']:
layer_name = MediaBlob.ParseType(layer['mediaType'])
if layer_name != 'config':
rv.layers[layer_name] = MediaBlob.Parse(layer)
return rv
@classmethod
def path(self, name):
name = str.replace(name, ':', os.path.sep)
return os.path.join(Manifest.LibPath(), name)
def to_dict(self):
layers: dict[str, MediaBlob] = {}
for layer in self.layers.values():
layers[layer.type] = layer.to_dict()
return {
'name': self.name,
'path': self.path(self.name),
'config': self.config.to_dict(),
'layers': layers,
}
def SaveParams(self, params):
filename = self.layers['params'].path()
backup = f"{filename}~"
# backup original params first time
if not os.path.exists(backup):
shutil.copy2(filename, backup)
saveJSON(filename, params)
def print(self):
print(json.dumps(self.to_dict(), indent=2))
if __name__ == '__main__':
if len(sys.argv) == 1:
print(f"usage: {sys.argv[0]} command")
print("list [ctx|sha]")
print("read model_name")
print("setctx model_name value")
print("restore_params model_name")
sys.exit()
if len(sys.argv) >= 2 and sys.argv[1] == 'list':
if len(sys.argv) == 3:
opt = sys.argv[2]
else:
opt = None
rv = Manifest.List(opt)
print('\n'.join(rv))
elif len(sys.argv) == 3 and sys.argv[1] == 'read':
name = sys.argv[2]
manifest = Manifest.Load(name)
manifest.print()
elif len(sys.argv) == 4 and sys.argv[1] == 'setctx':
name = sys.argv[2]
value = int(sys.argv[3])
manifest = Manifest.Load(name)
params = manifest.layers['params'].content()
params['num_ctx'] = value
manifest.SaveParams(params)
elif len(sys.argv) == 3 and sys.argv[1] == 'restore_params':
name = sys.argv[2]
manifest = Manifest.Load(name)
path = manifest.layers['params'].path()
backup = f"{path}~"
if not os.path.exists(backup):
print("no backup to restore from, running setctx will automatically create a backup")
else:
print(f"restoring params from {backup}")
shutil.copy2(backup, path)
@jocot
Copy link
Author

jocot commented Jan 7, 2024

Ollama python utility script

This is a script I wrote to view and edit the default model context size setting (num_ctx)

It can be used to fix models that have the wrong context size set, which display the following error:
Error: llama runner process has terminated

It assumes your ollama user folder is located in /home/ollama

Save olu.py to any folder and open a terminal in that folder.

Commands:

List all installed models
./olu.py list

Only list installed models with a context size (num_ctx) set.
If a model does not appear, then ollama will use the setting thats hard coded into the model (usually 2048 for llama based models).
./olu.py list ctx

Example

$ ./olu.py list ctx
dl:orangetin-openhermes-mixtral-8x7b.Q4_K_M 32768
dl:orca2-13b-16k-q8_0 16384
dl:starling-lm-7b-alpha.Q8_0 4096
dolphin-mixtral:8x7b-v2.6-q6_K 16384
llama2:70b 4096
mixtral:8x7b-instruct-v0.1-q4_K_M 32768
openchat:7b-v3.5-q6_K 4096
orca2:13b-q8_0 8192
wizardcoder:34b-python 16384
yarn-mistral:7b-128k-q6_K 131072

List installed models with their sha256 hash
./olu.py list sha

View model configuration metadata
./olu.py read dolphin-mixtral

Change the num_ctx setting for dolphin-mixtral.
Parameters are backed up automatically the first time this is run on a model.
sudo ./olu.py setctx dolphin-mixtral 16384

Restore the original parameters (only if setctx has been used previously)
sudo ./olu.py restore_params dolphin-mixtral

Limitations
sudo is required to change or restore settings.

Due to the way Ollama stores it's configuration, some models may share the same parameters file if they have the same base model.
If you change num_ctx for one, it'll apply to all models using the same parameters file.

eg.
codellama:34b-instruct and codellama:34b-instruct-q6_K

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment