Skip to content

Instantly share code, notes, and snippets.

@lontivero
Forked from mausch/flake.nix
Created November 6, 2023 11:58
Show Gist options
  • Save lontivero/cd15722ad94cf2ace1db93dce5c0d37d to your computer and use it in GitHub Desktop.
Save lontivero/cd15722ad94cf2ace1db93dce5c0d37d to your computer and use it in GitHub Desktop.
llama-vicuna.nix
{
description = "llama.cpp running vicuna";
inputs = {
llama.url = "github:ggerganov/llama.cpp/aaf3b23debc1fe1a06733c8c6468fb84233cc44f";
flake-utils.url = "github:numtide/flake-utils/033b9f258ca96a10e543d4442071f614dc3f8412";
nixpkgs.url = "github:NixOS/nixpkgs/d9f759f2ea8d265d974a6e1259bd510ac5844c5d";
};
outputs = { self, flake-utils, llama, nixpkgs }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
vicuna = pkgs.fetchurl {
url = "https://huggingface.co/eachadea/ggml-vicuna-13b-4bit/resolve/main/ggml-vicuna-13b-4bit-rev1.bin";
sha256 = "sha256-EpfAMUCnlfHmJyjI3ilUinnp0Aoych1j0tutOqmMdO0=";
};
in
{
packages.default = pkgs.writeScriptBin "llama-vicuna" ''
${llama.packages.${system}.default}/bin/llama -m ${vicuna} --color --batch_size 256 --n_predict -1 --top_k 12 --top_p 1 --temp 0 --repeat_penalty 1.05 --ctx_size 2048 --instruct --reverse-prompt '### Human:'
'';
apps.default = {
type = "app";
program = "${self.packages.${system}.default}/bin/llama-vicuna";
};
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment