llm.el

;; -*- lexical-binding: t; -*-
(require 'url)

(defun llm (command)
  (interactive "sWhat: ")
  (let* ((start (if (use-region-p) (region-beginning) (point)))
         (end (if (use-region-p) (region-end) (point)))
         (text-before (buffer-substring-no-properties (point-min) start))
         (selection (buffer-substring-no-properties start end))
         (text-after (buffer-substring-no-properties end (point-max))))
    (request "http://localhost:4242/completion"
      :type "POST"
      :data `(("before" . ,text-before)
	      ("after" . ,text-after)
	      ("command" . ,command)
	      ("selection" . ,selection))
      :parser 'json-read
      :success (cl-function
		(lambda (&key data &allow-other-keys)
		  (let ((text (alist-get 'text data)))
		    (kill-region start end)
		    (insert text))))
      :error (cl-function
              (lambda (&rest args &key error-thrown &allow-other-keys)
                (message "Failed with error: %S" error-thrown))))))

(global-set-key (kbd "C-`") 'llm)

llmserve.lisp

;; Define the API key
(defparameter *api-key* "get openrouter or openai api key")

(import 'alexandria:assoc-value)

(defun run-llm (prompt content)
  (let ((json-payload
	  (json:encode-json-to-string
           `(("model" . "google/gemini-flash-1.5")
             ("messages" . ((("role" . "system")
                              ("content" . ,prompt))
                             (("role" . "user")
                              ("content" . ,content))))
             ("max_tokens" . 4000)))))
    (multiple-value-bind (body status)
       ;; compatible with openai API, just change the URL
	(drakma:http-request "https://openrouter.ai/api/v1/chat/completions"
			     :method :post
			     :additional-headers `(("Authorization" . ,(format nil "Bearer ~a" *api-key*)))
			     :content-type "application/json"
			     :content json-payload)
      (if (= status 200)
	  (let ((json (json:decode-json-from-string
		       (flexi-streams:octets-to-string body))))
	    (identity (assoc-value
		       (assoc-value (first (assoc-value json :choices)) :message)
		       :content)))))))

(hunchentoot:start (make-instance 'hunchentoot:easy-acceptor :port 4242))

(hunchentoot:define-easy-handler (completion :uri "/completion") (before after command selection)
  (setf (hunchentoot:content-type*) "text/plain")
  (princ
   (run-llm (format nil "You're an assistant helping user to write text.
Current text in the buffer is defined in `before`, `selection` and `after`. Selection might be empty.
Empty command should be interpreted as: just write some text, as would be reasonable.

Return JSON document like this:

{ \"interpretation\": \"<what do you think this request is about\",
  \"text\": \"<text which would replace the selection (or is insert after `before`>\"
}

Do not add anything before the JSON. Do not include text which is already present in `before` or `after`.
")
	    (json:encode-json-to-string `(("before" . ,before)
					  ("after" . ,after)
					  ("selection" . ,selection)
					  ("command" . ,command))))))