llama.cpp function calling test

fn.js

const axios = require('axios').default;

const GRAMMAR = `
root   ::= response | function

response  ::= "response" ": " ([^\r]*)

function  ::= "function" ": {" (
    ws "\\"name\\":" ws string ","
    ws "\\"arguments\\":" ws object
  ) "}"

value  ::= object | array | string | number | ("true" | "false" | "null") ws

object ::=
  "{" ws (
            string ":" ws value
    ("," ws string ":" ws value)*
  )? "}" ws

array  ::=
  "[" ws (
            value
    ("," ws value)*
  )? "]" ws

string ::=
  "\\"" (
    [^"\\\\] |
    "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
  )* "\\"" ws

number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws

# Optional space: by convention, applied in this grammar after literal chars when allowed
ws ::= ([ \\t\\n] ws)?
`.trim();

const TOOLS = [
  {
    "type": "function",
    "function": {
      "name": "get_current_weather",
      "description": "Get the current weather",
      "parameters": {
        "type": "object",
        "properties": {
          "location": {
            "type": "string",
            "description": "The city and state, e.g. San Francisco, CA",
          },
        },
        "required": ["location"],
      },
    }
  }
];

(async () => {
  const response = await axios.post(
    'http://127.0.0.1:8080/v1/chat/completions',
    {
      messages: [
        { role: 'system', content: 'Your response start with the word "response" if you want to send a text response. Else, start your response with the word "function" to call a function.' },
        //{ role: 'user', content: 'hi' },
        { role: 'user', content: `What's the weather in Paris?\n\nFunction that you can use:\n\n${JSON.stringify(TOOLS[0], null, 2)}` },
      ],
      grammar: GRAMMAR,
      stream: true,
    },
    {
      responseType: 'stream',
      validateStatus: () => true,
      timeout: 999999999, // never timeout
    }
  );
  response.data.on('data', (chunk) => {
    const str = chunk.toString();
    const receivedJson = str.substring(str.indexOf('{'), str.lastIndexOf('}') + 1);
    const data = JSON.parse(receivedJson);
    const delta = data.choices[0].delta.content;
    if (delta) {
      process.stdout.write(data.choices[0].delta.content);
    }
  });

  response.data.on('end', () => {
    console.log('\n\nfinished');
  });
})();