Llama.cpp + Alpaca-30B

flake.nix

{
  description = "llama.cpp running Alpaca-30B";

  inputs = {
    llama.url = "github:ggerganov/llama.cpp/aaf3b23debc1fe1a06733c8c6468fb84233cc44f";
    flake-utils.url = "github:numtide/flake-utils/033b9f258ca96a10e543d4442071f614dc3f8412";
    nixpkgs.url = "github:NixOS/nixpkgs/d9f759f2ea8d265d974a6e1259bd510ac5844c5d";
  };  

  outputs = { self, flake-utils, llama, nixpkgs }: 
    flake-utils.lib.eachDefaultSystem (system:
      let
      	pkgs = import nixpkgs { inherit system; };
        alpaca = pkgs.fetchurl {
          url = "https://huggingface.co/Pi3141/alpaca-lora-30B-ggml/resolve/main/ggml-model-q4_1.bin";
          sha256 = "sha256-+0HvOhmdGuZbIkH/maG0fUoqUphBcP6EMgPWn3BKsDg=";
        };
      in
      {
        packages.default = pkgs.writeScriptBin "llama-vicuna" ''
          ${llama.packages.${system}.default}/bin/llama -m ${alpaca} --color --instruct
          # --batch_size 256 --n_predict -1 --top_k 12 --top_p 1 --temp 0 --repeat_penalty 1.05 --ctx_size 2048
        '';

        apps.default = {
          type = "app";
          program = "${self.packages.${system}.default}/bin/llama-vicuna";
        };
      });
}