A 100% local and typescript LLM in Apple Shortcut to be used everywhere on your computer in <1 second

package.json

{
  "dependencies": {
    "@llama-node/llama-cpp": "^0.0.32",
    "llama-node": "^0.0.32"
  },
  "devDependencies": {
    "@types/node": "^18.16.3"
  }
}

run.ts

import { LLM } from "llama-node";
import { LLamaCpp } from "llama-node/dist/llm/llama-cpp.js";
import { exec } from "child_process";

const model = "./ggml-vicuna-7b-4bit-rev1.bin";
const llama = new LLM(LLamaCpp);
const config = {
    path: model,
    enableLogging: true,
    nCtx: 1024,
    nParts: -1,
    seed: 0,
    f16Kv: false,
    logitsAll: false,
    vocabOnly: false,
    useMlock: false,
    embedding: false,
    useMmap: true,
};
llama.load(config);

// get from the command argument
const humanInstruction = process.argv[2];
const prompt = `A chat between a user and an assistant. The user is a human and the assistant is a computer.
The assistant obeys the following rules:
- The assistant must respond concisely and informatively.
- The assistant must end its response with "###".
USER: ${humanInstruction}
ASSISTANT:`;

console.log("prompt", prompt);

process.on("SIGINT", () => {
    console.log("My master has commanded me to terminate. Goodbye.");
    process.exit(0);
});

let allText = "";
llama.createCompletion({
    prompt,
    nThreads: 12,
    nTokPredict: 1048,
    topK: 40,
    topP: 0.1,
    temp: 0.7,
    repeatPenalty: 1,
}, (response) => {
    const outputText = response.token;
    allText += outputText;
    const escapedText = outputText.replace(/"/g, '\\"');
    const appleScript = `tell application \\"System Events\\" to keystroke \\"${escapedText}\\"`;
    // setTimeout(() => process.exit(0), 5000);

    exec(`osascript -e "${appleScript}"`, (error, stdout, stderr) => { // Use double quotes here
        if (error) {
            console.error(`Error: ${error.message}`);
            return;
        }
        if (stderr) {
            console.error(`Stderr: ${stderr}`);
            return;
        }
        if (allText.includes("###")) {
            console.log("My mission is complete, auto-terminating.");
            process.exit(0);
        }
    });
});

1. download a model from somewhere https://huggingface.co/models?search=vicuna or https://huggingface.co/models?search=llama
2. npm i
3. npx tsx run.ts "what is a neural turing machine?"
4. (optional) install raycast
5. use this shortcut (tweak ur path) https://www.icloud.com/shortcuts/fdf5f6ce42914c4e8088aec79a90aada
   • Open Obsidian (or any text editor) - Say Hey Siri run AI Stream - What is a neural turing machine?
   • Or option+space bar+AI Stream+ fill your prompt

Thank you.