llama-vicuna.nix

flake.nix

{
  description = "llama.cpp running vicuna";

  inputs = {
    llama.url = "github:ggerganov/llama.cpp/aaf3b23debc1fe1a06733c8c6468fb84233cc44f";
    flake-utils.url = "github:numtide/flake-utils/033b9f258ca96a10e543d4442071f614dc3f8412";
    nixpkgs.url = "github:NixOS/nixpkgs/d9f759f2ea8d265d974a6e1259bd510ac5844c5d";
  };  

  outputs = { self, flake-utils, llama, nixpkgs }: 
    flake-utils.lib.eachDefaultSystem (system:
      let
      	pkgs = import nixpkgs { inherit system; };
        vicuna = pkgs.fetchurl {
          url = "https://huggingface.co/eachadea/ggml-vicuna-13b-4bit/resolve/main/ggml-vicuna-13b-4bit-rev1.bin";
          sha256 = "sha256-EpfAMUCnlfHmJyjI3ilUinnp0Aoych1j0tutOqmMdO0=";
        };

         flags = builtins.mapAttrs (_: builtins.toString) {
            batch_size = 256;
            n_predict = -1;
            top_k = 12;
            top_p = 1;
            temp = 0;
            repeat_penalty = 1.05;
            ctx_size = 2048;

            reverse-prompt = "'### Human:'";
        };
      in
      {
        packages.default = pkgs.writeScriptBin "llama-vicuna" ''
          ${llama.packages.${system}.default}/bin/llama -m ${vicuna} --color --batch_size ${flags.batch_size} --n_predict ${flags.n_predict} --top_k ${flags.top_k} --top_p ${flags.top_p} --temp ${flags.temp} --repeat_penalty ${flags.repeat_penalty} --ctx_size ${flags.ctx_size} --instruct --reverse-prompt ${flags.reverse-prompt}
        '';

        apps.default = {
          type = "app";
          program = "${self.packages.${system}.default}/bin/llama-vicuna";
        };
      });
}

Make easier to parametrize