[llama.cpp on Nix] #nix #llm

readme.md

Get clblast.nix from nix-unstable

Get a model: https://huggingface.co/TheBloke/WizardLM-13B-V1.2-GGML/blob/main/wizardlm-13b-v1.2.ggmlv3.q4_1.bin

Build: nix-build

Example prompt:

./result/bin/main -m ~/Downloads/wizardlm-13b-v1.2.ggmlv3.q4_1.bin -n 128 --prompt "Unce upon a time"

default.nix

{ pkgs ? import <nixpkgs> {} }:

let
  clblast = pkgs.callPackage ./clblast {};
in

pkgs.callPackage ./llama { inherit clblast; }

llama.nix

{ lib
, stdenv
, fetchFromGitHub
, cmake
, ninja
, clblast
, ocl-icd
, python3
}:

stdenv.mkDerivation rec {
  pname = "llama.cpp";
  version = "b5472ea";

  src = fetchFromGitHub {
    owner = "ggerganov";
    repo = "llama.cpp";
    rev = "master-b5472ea";
    hash = "sha256-O96tIt9F3tZMEEG3FP+zKe7NX05+m4gU+iZaFElEl4s=";
  };

  nativeBuildInputs = [
    cmake
    ninja
  ];

  buildInputs = [
    clblast
    ocl-icd
    (python3.withPackages (pkgs: with pkgs; [ numpy sentencepiece ]))
  ];

  cmakeFlags = [
    "-DLLAMA_CLBLAST=ON"
  ];
}