REPL session of babashka and sci internals @ London Clojurians December 2020

repl.clj

(ns ldnclj.repl
  (:require [babashka.main :as bb]
            [cheshire.core :as json]
            [edamame.core :as e]
            [sci.core :as sci]
            [sci.impl.analyzer :as ana]
            [sci.impl.interop :as interop]
            [sci.impl.interpreter :as i]
            [sci.impl.parser :as p]))

;; Notes
(comment
  ;; REPL:
  ;; In babashka project: "lein with-profiles +test repl"
  ;; Useful for hiding/showing blocks:
  ;; hs-minor-mode, hs-show-block, hs-hide-block
  )

;; Intro
(comment
  ;; See slides:
  ;; https://speakerdeck.com/borkdude/babashka-and-sci-internals-at-london-clojurians-december-2020
  )

;; Usage of bb and sci API
(comment
  ;; Normally we call bb as a CLI app, but today we will look at the
  ;; internals from within a REPL.

  ;;;; Exit code ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  ;; (require '[babashka.main :as bb])
  (bb/main "-e" "(+ 1 2 3)") ;;=> 0, the exit code

  (with-out-str (bb/main "(+ 1 2 3)")) ;;=> "6\n", printed to stdout

  ;; Babashka uses sci to interpret code
  ;; (require '[sci.core :as sci])
  (sci/eval-string "(+ 1 2 3)") ;;=> 6

  (bb/main "(/ 1 0)") ;;=> 1, non-zero exit code

  (with-out-str
    (binding [*err* *out*]
      (bb/main "(/ 1 0)"))) ;;=> Divide by zero

  ;; Whoops, exception:
  (sci/eval-string "(/ 1 0)")

  ;;;; Interop ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  ;; Babashka can do interop
  (with-out-str (bb/main "(java.io.File. \".\")")) ;;=> "#object[java.io.File 0x70b70ff5 \".\"]\n"
  (sci/eval-string "(java.io.File. \".\")") ;; unable to resolve classname java.io.File
  ;; We need to explicitly add classes. This works:
  (sci/eval-string "(java.io.File. \".\")" {:classes {'java.io.File java.io.File}})

  ;;;; Libraries ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  ;; Babashka has libs built in
  (with-out-str (bb/main "
(require '[cheshire.core :as json])
(json/parse-string (json/generate-string {:a 1}) true)")) ;;=> "{:a 1}\n"

  (sci/eval-string "
(require '[cheshire.core :as json])
(json/parse-string (json/generate-string {:a 1}) true)") ;;=> Could not find namespace cheshire.core

  ;; (require '[cheshire.core :as json])
  ;; This works:
  (sci/eval-string "
(require '[cheshire.core :as json])
(json/parse-string (json/generate-string {:a 1}) true)"
                   {:namespaces {'cheshire.core {'generate-string json/generate-string
                                                 'parse-string    json/parse-string}}})

  ;;;;; Vars ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  (sci/eval-string "user/x"
                   {:namespaces {'user {'x 1}}}) ;;=> 1

  ;; This doesn't work, because 1 isn't a var in sci
  (sci/eval-string "(alter-var-root #'user/x (constantly 2))"
                   {:namespaces {'user {'x 1}}})

  ;; This does work because def creates a sci var
  (sci/eval-string "
(def x 1)
(alter-var-root #'user/x (constantly 2))") ;;=> 2

  ;; How do we create sci vars to pass via opts?
  (def x (sci/new-var 'x 1))
  (sci/eval-string "(alter-var-root #'user/x (constantly 2))"
                   {:namespaces {'user {'x x}}}) ;;=> 2

  ;; How about dynamic vars?
  (sci/eval-string "(binding [x 10] x)"
                   {:namespaces {'user {'x x}}}) ;; Error, x is not dynamic

  (def x-dynamic (sci/new-dynamic-var 'x 1))
  (sci/eval-string "(binding [*x* 10] *x*)"
                   {:namespaces {'user {'*x* x-dynamic}}}) ;;=> 10

  ;; What about macros?
  (sci/with-out-str (sci/eval-string "(defmacro foo [x] `(do ~x ~x)) (foo (prn :x))"))
  ;;=> ":x\n:x\n"

  (sci/with-out-str (sci/eval-string "(foo (prn :x))"
                                     {:namespaces {'user {'foo (fn [x] `(do ~x ~x))}}}))
  
  ;;=> ":x\n"
  ;; Hmm, only one :x, why?

  ;; Metadata to mark fn as macro:
  (sci/with-out-str (sci/eval-string
                     "(foo (prn :x))"
                     {:namespaces {'user {'foo ^:sci/macro (fn [_form _env x]
                                                             ;; inspect form!
                                                             ;; (prn _form)
                                                             `(do ~x ~x))}}}))

  ;; Create a macro contained in sci var:
  (def foo (sci/new-macro-var 'foo (fn [_form _env x]
                                     ;; inspect form!
                                     ;; (prn _form)
                                     `(do ~x ~x))))
  (sci/with-out-str (sci/eval-string
                     "(foo (prn :x))"
                     {:namespaces {'user {'foo foo}}}))


  ;; Copy existing functions and macros into sci vars:
  (defmacro foo* [x] `(do ~x ~x))
  (macroexpand '(foo* (prn :x))) ;; => (do (prn :x) (prn :x))
  ;; Let's copy using sci/copy-var, which needs a sci ns.
  (def user-ns (sci/create-ns 'user))
  (def sci-macro-var (sci/copy-var foo* user-ns))
  (sci/with-out-str (sci/eval-string
                     "(foo (prn :x))"
                     {:namespaces {'user {'foo sci-macro-var}}}))
  ;;=> ":x\n:x\n"
  ::fin)

;; Creating a sci-based REPL
(comment
  ;; In a REPL we need to remember context as we evaluate form after form.
  (def ctx (sci/init {}))
  (sci/eval-string* ctx "(def x 1)")
  (sci/eval-string* ctx "x") ;; => 1

  ;; Collecting input
  (sci/eval-string (read-line))

  ;; Read-line is not enough: keep reading until form is complete.
  (def rdr (sci/reader *in*))

  ;; Parsing a complete form from stdin:
  (sci/parse-next ctx rdr)
  (with-in-str "(+ 1 2\n\n3)" (sci/parse-next ctx (sci/reader *in*))) ;; ;;=> (+ 1 2 3)

  ;; How to evaluate?
  (sci/eval-form ctx '(+ 1 2 3)) ;;=> 6

  ;; All together now:
  (with-in-str "(+ 1 2\n\n3)" (->> (sci/reader *in*)
                                   (sci/parse-next ctx )
                                   (sci/eval-form ctx))) ;; => 6

  (defn repl []
    (let [next-form (sci/parse-next ctx (sci/reader *in*))
          next-val (sci/eval-form ctx next-form)]
      (when-not (identical? :repl/quit next-val)
        (prn next-form '-> next-val)
        (repl))))

  (repl)
  )

;; Parsing forms
(comment
  ;; Edamame is the lib that sci uses to parse Clojure forms
  ;; It is GraalVM-compatible (no eval) and attaches location metadata to forms
  (e/parse-string "{:a 1}") ;;=> {:a 1}
  (meta (e/parse-string "{:a 1}")) ;;=> {:row 1, :col 1, :end-row 1, :end-col 7}

  ;; Resolving namespaces
  (e/parse-string "::foo") ;; ERROR, should use :auto-resolve + :current to resolve ns
  (e/parse-string "::foo" {:auto-resolve {:current 'foo}}) ;;=> :foo/foo
  (e/parse-string "::s/foo" {:auto-resolve {'s 'clojure.string}}) ;;=> clojure.string/foo

  ;; Reader conditionals
  (e/parse-string "#?(:bb 1 :clj 2)" {:read-cond true}) ;;=> nil
  (e/parse-string "#?(:bb 1 :clj 2)" {:read-cond true :features #{:bb}}) ;;=> 1

  ;; Syntax quote
  (eval (e/parse-string "`[1 2 x]" {:syntax-quote true})) ;; [1 2 x]
  (eval (e/parse-string "`[1 2 x]"
                        {:syntax-quote {:resolve-symbol (fn [_] 'foobar/x)}}))
  ;; => [1 2 foobar/x]
  ;; So this is basically what is going on in sci.impl.parser
  ::fin)

;; Inside sci
(comment
  ;; (require '[sci.impl.interpreter :as i])
  ;; (require '[sci.impl.analyzer :as ana])
  ;; (require '[sci.impl.parser :as p])
  
  #_{:clj-kondo/ignore [:redefined-var]}
  (def ctx (sci/init {}))

  ;; The analyzer's job is to inspect and enrich forms with instructions for the
  ;; interpreter
  (ana/analyze ctx (sci/parse-string ctx "x")) ;; could not resolve symbol x at
                                               ;; line 1, column 1...

  (def analyzed-fn (ana/analyze ctx (p/parse-string ctx "(fn [] 1)")))
  (meta analyzed-fn) ;;=> :sci.impl/op :fn
  ;; sci.impl/op decides what the interpreter is going to do with the form
  (def interpreted-fn (i/interpret ctx analyzed-fn))
  (interpreted-fn) ;;=> 1

  (def analyzed-def
    (ana/analyze ctx (sci/parse-string ctx "(def x 1)"))) ;;=> (def x 1)

  (meta analyzed-def) ;;=> :sci.impl/op :call

  (i/interpret ctx analyzed-def) ;; now x is defined...

  (type (i/interpret ctx analyzed-def)) ;;=> sci.impl.vars.Var

  ;; deref the var:
  (i/interpret ctx (ana/analyze ctx (p/parse-string ctx "x"))) ;;=> 1
  (sci/eval-string* ctx "x") ;;=> 1
  
  ;; Let's define a macro:
  (i/eval-string* ctx "(defmacro foo [] `(+ 1 2 3))")

  ;; The analyzer also does macro-expansion:
  (def analyzed-expr (ana/analyze ctx (p/parse-string ctx "(foo)")))
  (meta analyzed-expr) ;; :sci.impl/op :call
  (i/interpret ctx analyzed-expr) ;;=> 6

  java.io.File/separator
  #_{:clj-kondo/ignore [:redefined-var]}
  (def ctx (sci/init {:classes {'java.io.File java.io.File}}))
  (def analyzed-static-field
    (ana/analyze ctx (p/parse-string ctx "java.io.File/separator")))
  ;;=> [java.io.File separator]
  (meta analyzed-static-field) ;;=> :sci.impl/op :static-access
  (i/interpret ctx analyzed-static-field) ;;=> "/"
  (interop/get-static-field analyzed-static-field) ;;=> "/"

  ;; special form
  (i/interpret ctx (ana/analyze ctx '(if 1 2 3))) ;;=> 2
  (i/eval-special-call ctx 'if '(if 1 2 3)) ;;=> 2
  (i/eval-if ctx '(if 1 2 3))
  (i/eval-if ctx '(1 (prn :foo) (prn :bar))) ;; (prn :foo) ;; no eval
  ;; doesn't work, symbol prn should have been resolved to var already in analyzer
  (i/eval-if ctx '(1 ^{:sci.impl/op :call} (prn :foo)))
  (i/eval-if ctx `(1 ^{:sci.impl/op :call} (~prn :foo))) ;; prints :foo

  #_{:clj-kondo/ignore [:redefined-var]}
  (def ctx (sci/init
             {:namespaces
              {'clojure.core
               {'prn (sci/copy-var prn (sci/create-ns 'clojure.core nil))}}}))
  (def analyzed-call (ana/analyze ctx '(prn :foo)))
  (i/eval-if ctx `(1 ~analyzed-call :else))
  (i/eval-if ctx `(false ~analyzed-call :else)) ;; => else
  )

;; Misc. topics, time permitting
(comment
  ;; GraalVM compilation, reflection config
  ;; Multimethods, protocols, reify
  )

(comment

  (sci/eval-string "#foo/bar [1 2 3]" {:readers {'foo/bar identity}})

  (sci/eval-string "#_[1 2 ]")
  )