llama-vicuna.nix

flake.nix

{
  description = "llama.cpp running vicuna";

  inputs = {
    llama.url = "github:ggerganov/llama.cpp/aaf3b23debc1fe1a06733c8c6468fb84233cc44f";
    flake-utils.url = "github:numtide/flake-utils/033b9f258ca96a10e543d4442071f614dc3f8412";
    nixpkgs.url = "github:NixOS/nixpkgs/d9f759f2ea8d265d974a6e1259bd510ac5844c5d";
  };  

  outputs = { self, flake-utils, llama, nixpkgs }: 
    flake-utils.lib.eachDefaultSystem (system:
      let
      	pkgs = import nixpkgs { inherit system; };
        vicuna = pkgs.fetchurl {
          url = "https://huggingface.co/eachadea/ggml-vicuna-13b-4bit/resolve/main/ggml-vicuna-13b-4bit-rev1.bin";
          sha256 = "sha256-EpfAMUCnlfHmJyjI3ilUinnp0Aoych1j0tutOqmMdO0=";
        };
      in
      {
        packages.default = pkgs.writeScriptBin "llama-vicuna" ''
          ${llama.packages.${system}.default}/bin/llama -m ${vicuna} --color --batch_size 256 --n_predict -1 --top_k 12 --top_p 1 --temp 0 --repeat_penalty 1.05 --ctx_size 2048 --instruct --reverse-prompt '### Human:'
        '';

        apps.default = {
          type = "app";
          program = "${self.packages.${system}.default}/bin/llama-vicuna";
        };
      });
}

nix run --refresh --no-write-lock-file git+https://gist.github.com/8d3e55926c2f0966e3b93defc44b65d5