Dynlib based Nim REPL using compiler API, clean up a bit

dynlib_based_nim_repl_clean.nim

import std / [strutils, strformat, tables, dynlib, os]
import noise, shell


import compiler/[llstream, renderer, types, magicsys, ast,
                 transf, # for code transformation (for -> while etc)
                 injectdestructors, # destructor injection
                 pathutils, # AbsoluteDir
                 modulegraphs] # getBody
import ./nimeval_dynlib_clean

# probably need to import `pragmas` and `wordrecg` to get
# `hasPragma` working

import hnimast
import typetraits

proc setupInterpreter(moduleName = "/t/script.nim"): Interpreter =
  let std = findNimStdLibCompileTime()
  var paths = newSeq[string]()
  paths.add std
  paths.add std & "/pure"
  paths.add std & "/core"
  paths.add std & "/pure/collections"
  paths.add std & "/posix"
  paths.add "/home/basti/.nimble/pkgs"
  #paths.add "/home/basti/CastData/ExternCode/units/src"
  result = createInterpreter(moduleName, paths, defines = @[])

proc printHelp() = echo ""

const procTmpl = """
{.push cdecl, exportc, dynlib.}
$#
{.pop.}
"""
const exprTmpl = """
$# # <- insert code to importc &
{.push cdecl, exportc, dynlib.}
proc tmp() =
  $#
{.pop.}
"""

type
  Repl = object
    intr: Interpreter
    #ctx: JitContext
    # Table of all precompiled functions
    # Maps function name to the compiled result ptr
    fnTab: Table[string, (string, string)]
    imports: string
    #stream: PLLStream
    #streamOpened = false
    buffer: string

  InputKind = enum
    ikProcDef, ikStatement, ikImport, ikExpression


proc callTmp(fname, procName: string) =
  ## XXX: do not unload lib!
  echo "Loading: ", fname, " name: ", procName
  let lib = loadLib(fname)
  doAssert lib != nil
  let foo = cast[(proc() {.nimcall.})](lib.symAddr(procName))
  doAssert foo != nil
  echo "Succesfully loaded foo : ", foo != nil
  foo()
  unloadLib(lib)

# maps known functions to their dynlib
#var fnTab = initTable[string, (string, string)]()

proc loadIt(fn, signature, lib: string): string =
  result = signature & "{.importc: \"" & fn & "\", dynlib: \"" & lib & "\".}"

proc codeToLoad(repl: Repl): string =
  for fn, (file, signature) in repl.fnTab:
    result.add loadIt(fn, signature, file) & "\n"

proc inputKind(line: string): InputKind =
  ## XXX: this will be improved obv
  if line.strip.startsWith("proc"): result = ikProcDef
  # we need to ask the nim compiler what the resulting type is!
  # Question: how do we deal with the *nim compiler* knowing the state? I.e. referencing a variable
  # `foo` 10 REPL statements after? Does that happen "automatically"?
  #elif
  elif line.strip.startsWith("import"): result = ikImport
  else: result = ikStatement

proc getFnName(line: string): string =
  result = line.strip()
  result.removePrefix("proc")
  let idx = result.find("(")
  result.delete(idx, result.len)
  result = result.strip()
  echo "FN NAME: ", result, " from ", line

proc withStream(intr: var Interpreter, code, outfile: string): string =
  let stream = llStreamOpen(code)
  intr.evalScript(stream, outfile)
  llStreamClose(stream)
  result = outfile.parentDir / "lib" & outfile.extractFilename.replace(".nim", ".so")

var counter = 0
proc writeCompile(repl: var Repl, fn, content: string): string =
  let fname = "tmp_file_$#.nim" % $counter
  let file = "/t/$#" % fname
  writeFile(file, content)
  echo "\tWrote:\n", content
  inc counter
  # compile as lib
  shell:
    nim c "--app:lib --verbosity:0" ($file)
  result = ("/t/lib" & fname).replace(".nim", ".so")

proc onlyWrite(repl: var Repl, fn, content: string): string =
  #let fname = "tmp_file_$#.nim" % $counter
  let fname = "script.nim" #"tmp_file.nim"
  let file = "/t/$#" % fname
  writeFile(file, content)
  echo "\tWrote:\n", content
  #inc counter
  result = file

proc handleProcDef(repl: var Repl, line: string) =
  ## MORE STUFF
  let fnName = getFnName(line)

  # make sure `fn` exported
  var line = line
  if "*" notin line:
    line = line.replace(fnName, fnName & "*")
  repl.buffer.add line & "\n"
  # signature
  var signature = line
  signature = signature.split("=")[0] ## XXX: Better extract real proc signature!!!

  let content = procTmpl % line
  let outfile = repl.onlyWrite(fnName, content)
  let libfile = repl.intr.withStream(content, outfile)

  let newname = libfile.replace(".so", "_" & $counter & ".so")
  copyFile(libfile, newname)
  inc counter

  #callTmp(libfile, fnName)

  repl.fnTab[fnName] = (newname, signature)

  #withReplStream(repl.buffer)
  #let t = repl.intr.selectRoutine(fnName)
  #echo "Jit it"
  #repl.compileOnly(t.ast, fnName)

proc handleStatement(repl: var Repl, line: string) =
  ## Statement: place in temporary proc and jit & run
  #var gn = "wrapper_fn_" & $counter
  #inc counter
  #var body = &"{repl.imports}\nproc {gn}*() =\n  {line}"
  #echo "Body: ", body
  #repl.buffer.add body & "\n"
  let imports = repl.imports & "\n"
  let loadCode = repl.codeToLoad()
  let content = exprTmpl % [imports & loadCode, line]

  #let outfile = repl.writeCompile("tmp", content)
  let outfile = repl.onlyWrite("tmp", content)
  let libfile = repl.intr.withStream(content, outfile)

  ## call tmp function
  callTmp(libfile, "tmp")

proc handleImport(repl: var Repl, line: string) =
  ## Imports for now are just appended to the import header, which is prefixed
  ## globally to an a statement
  repl.imports.add line & "\n"

proc handleUserInput(repl: var Repl, line: string) =
  # pass code through nim compiler
  case line.inputKind
  of ikProcDef:
    # just JIT compile the proc!
    repl.handleProcDef(line)
  of ikStatement:
    # JIT compile the body and run
    repl.handleStatement(line)
  of ikImport:
    # handle imports
    repl.handleImport(line)
  of ikExpression: doAssert false

proc repl(repl: var Repl) =
  var noise = Noise.init()

  let prompt = Styler.init(fgRed, "Red ", fgGreen, "nim> ")
  noise.setPrompt(prompt)

  when promptPreloadBuffer:
    noise.preloadBuffer("")

  when promptHistory:
    var file = "history"
    discard noise.historyLoad(file)

  when promptCompletion:
    proc completionHook(noise: var Noise, text: string): int =
      const words = ["apple", "diamond", "diadem", "diablo", "horse", "home", "quartz", "quit"]
      for w in words:
        if w.find(text) != -1:
          noise.addCompletion w

    noise.setCompletionHook(completionHook)

  while true:
    let ok = noise.readLine()
    if not ok: break

    ## XXX: figure out how to input multiple lines
    let line = noise.getLine
    case line
    of ".help": printHelp()
    of ".quit": break
    else:
      if line.len > 0:
        repl.handleUserInput(line.strip)

    when promptHistory:
      if line.len > 0:
        noise.historyAdd(line)
        discard noise.historySave(file)

  when promptHistory:
    discard noise.historySave(file)


proc setupRepl =
  echo "setting up interpreter"
  var intr = setupInterpreter()
  # add Unchained
  # [X] Adding at runtime works just fine after the interpreter is constructed!
  intr.graph.config.searchPaths.add(AbsoluteDir "/home/basti/CastData/ExternCode/units/src")
  ## ^--- this way we can add more paths when user imports libraries!
  ### XXX: add `extract import from user input` and then add those imports like this
  ## set up gcc jit context
  #let jitCtx = initJitContext(intr, true)
  #var repl = Repl(intr: intr, ctx: jitCtx)
  var repl = Repl(intr: intr, fnTab: initTable[string, (string, string)]())
  repl(repl)

proc main() =
  setupRepl()
when isMainModule:
  import cligen
  dispatch main

nimeval_dynlib_clean.nim

#
#
#           The Nim Compiler
#        (c) Copyright 2018 Andreas Rumpf
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

## exposes the Nim VM to clients.
import compiler / [
  ast, modules, condsyms,
  options, llstream, lineinfos, vm,
  vmdef, modulegraphs, idents, pathutils,
  scriptconfig, cgen, extccomp, cgendata, ropes,
  passes
]

import std/[compilesettings, os, tables]
import compiler / pipelines


when defined(nimPreviewSlimSystem):
  import std/[assertions, syncio]

type
  Interpreter* = ref object ## Use Nim as an interpreter with this object
    mainModule: PSym
    graph*: ModuleGraph
    scriptName: string
    idgen*: IdGenerator

iterator exportedSymbols*(i: Interpreter): PSym =
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
  for s in modulegraphs.allSyms(i.graph, i.mainModule):
    yield s

proc selectUniqueSymbol*(i: Interpreter; name: string;
                         symKinds: set[TSymKind] = {skLet, skVar}): PSym =
  ## Can be used to access a unique symbol of ``name`` and
  ## the given ``symKinds`` filter.
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
  let n = getIdent(i.graph.cache, name)
  var it: ModuleIter
  var s = initModuleIter(it, i.graph, i.mainModule, n)
  result = nil
  while s != nil:
    if s.kind in symKinds:
      if result == nil: result = s
      else: return nil # ambiguous
    s = nextModuleIter(it, i.graph)

proc selectRoutine*(i: Interpreter; name: string): PSym =
  ## Selects a declared routine (proc/func/etc) from the main module.
  ## The routine needs to have the export marker ``*``. The only matching
  ## routine is returned and ``nil`` if it is overloaded.
  result = selectUniqueSymbol(i, name, {skTemplate, skMacro, skFunc,
                                        skMethod, skProc, skConverter})

proc callRoutine*(i: Interpreter; routine: PSym; args: openArray[PNode]): PNode =
  assert i != nil
  result = vm.execProc(PCtx i.graph.vm, routine, args)

proc getGlobalValue*(i: Interpreter; letOrVar: PSym): PNode =
  result = vm.getGlobalValue(PCtx i.graph.vm, letOrVar)

proc setGlobalValue*(i: Interpreter; letOrVar: PSym, val: PNode) =
  ## Sets a global value to a given PNode, does not do any type checking.
  vm.setGlobalValue(PCtx i.graph.vm, letOrVar, val)

proc implementRoutine*(i: Interpreter; pkg, module, name: string;
                       impl: proc (a: VmArgs) {.closure, gcsafe.}) =
  assert i != nil
  let vm = PCtx(i.graph.vm)
  vm.registerCallback(pkg & "." & module & "." & name, impl)


proc findNimStdLib*(): string =
  ## Tries to find a path to a valid "system.nim" file.
  ## Returns "" on failure.
  try:
    let nimexe = os.findExe("nim")
      # this can't work with choosenim shims, refs https://github.com/dom96/choosenim/issues/189
      # it'd need `nim dump --dump.format:json . | jq -r .libpath`
      # which we should simplify as `nim dump --key:libpath`
    if nimexe.len == 0: return ""
    result = nimexe.splitPath()[0] /../ "lib"
    if not fileExists(result / "system.nim"):
      when defined(unix):
        result = nimexe.expandSymlink.splitPath()[0] /../ "lib"
        if not fileExists(result / "system.nim"): return ""
  except OSError, ValueError:
    return ""

proc findNimStdLibCompileTime*(): string =
  ## Same as `findNimStdLib` but uses source files used at compile time,
  ## and asserts on error.
  result = querySetting(libPath)
  doAssert fileExists(result / "system.nim"), "result:" & result


import std / [os, strutils]
var
  first = true
  ropesArray: array[TCFileSection, Rope]
  ropesTable = initTable[string, array[TCFileSection, Rope]]()

proc commandCompileToC(graph: ModuleGraph) =
  let conf = graph.config
  #extccomp.initVars(conf)
  setPipeLinePass(graph, SemPass)
  setPipeLinePass(graph, CGenPass)

  compileProject(graph)

  ## write the C code to files (in `~/.cache/nim/foo_r`)
  cgenWriteModules(graph.backend, conf)
  ## Call the C compiler to generate the shared lib
  extccomp.callCCompiler(conf)

  ## Other potential things I tried to "reset" without having to redo everything
  #graph.markDirty(conf.projectMainIdx)

  #graph.clearPasses()
  #graph.resetForBackend()
  # reset all modules also resets the main module, i.e. everything
  #graph.resetAllModules()

proc setupModuleGraph(conf: ConfigRef): ModuleGraph =
  var cache = newIdentCache()
  var graph = newModuleGraph(cache, conf)
  connectPipelineCallbacks(graph)
  result = graph

proc setupConfig(scriptName: string, searchPaths: openArray[string | AbsoluteDir]): ConfigRef =
  var conf = newConfigRef()
  initDefines(conf.symbols)
  #for define in defines: ## <-- don't define any symbols at the moment
  #  defineSymbol(conf.symbols, define[0], define[1])
  conf.selectedGC = gcOrc ## <-- this is what's needed to get destructors working!
  defineSymbol(conf.symbols, "gcorc")
  defineSymbol(conf.symbols, "gcdestructors")
  incl conf.globalOptions, optSeqDestructors
  incl conf.globalOptions, optTinyRtti
  incl conf.globalOptions, optGenDynLib ## <- we want to produce a shared lib, this is equivalent to `nim c --app:lib`
  defineSymbol(conf.symbols, "nimSeqsV2")
  defineSymbol(conf.symbols, "nimV2")
  defineSymbol(conf.symbols, "danger")
  defineSymbol(conf.symbols, "release")

  for p in searchPaths:
    conf.searchPaths.add(AbsoluteDir p)
    if conf.libpath.isEmpty: conf.libpath = AbsoluteDir p

  extccomp.initVars(conf)
  conf.outfile = RelativeFile scriptName
  result = conf

proc setupModule(scriptName: string, graph: ModuleGraph): (PSym, IdGenerator) =
  var m = graph.makeModule(scriptName)
  incl(m.flags, sfMainModule)    ## <-- I don't think we need this, but maybe we should after all
  var idgen = idGeneratorFromModule(m)
  var vm = newCtx(m, graph.cache, graph, idgen)
  #vm.mode = emRepl
  # vm.features = flags ## <-- We don't use any flags at the moment
  if true: # registerOps:
    vm.registerAdditionalOps() # Required to register parts of stdlib modules
  graph.vm = vm
  #setPipeLinePass(graph, SemPass) ## <-- This would only do the "nim compilation", i.e. processing macros, injecting destructors etc
  setPipeLinePass(graph, CgenPass) ## <-- this tells the compiler to do everything, incl compiling to C
  graph.compilePipelineSystemModule()

  result = (m, idgen)

proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil, file: string) =
  ## This can also be used to *reload* the script.

  ## All this commented out code is me trying to manually fix the state of the module graph
  ## after the first script evaluation to make it work, if we don't start from scratch.
  ## In particular the `s` field of the `BModule`, contained in the `BModuleList`, which is
  ## the `graph.backend` contain all the strings (`Rope` type in compiler) that actually
  ## just store the literal C code that is emitted in the headers, types etc. So to avoid
  ## duplicates, I tried to reset these to the state they were in after / before the first
  ## compilation
  #block:
  #  #i.graph.backend = newModuleList(i.graph)
  #  #var graph = newModuleGraph(i.graph.cache, i.graph.config)
  #  #connectPipelineCallbacks(graph)
  #
  #  #var backend = cast[BModuleList](graph.backend)
  #  #backend.mainModProcs = Rope""
  #  #backend.mainModInit = Rope""
  #  #backend.otherModsInit = Rope""
  #  #backend.mainDatInit = Rope""
  #  #graph.backend = backend
  #  echo i.graph.config.nimMainPrefix
  #
  #  for m in cgenModules(backend):
  #    m.s[cfsInitProc] = newRopeAppender()
  #  #  for x in mitems(m.s):
  #  #    x = Rope""
  #
  #  i.graph = graph
  #  #i.graph.config.nimMainPrefix = ""
  #  #i.graph.backend = backend
  #  var m = i.graph.makeModule(file.replace(".nim", "_script.nim"))
  #  #incl(m.flags, sfMainModule)
  #  var idgen = idGeneratorFromModule(m)
  #  var vm = newCtx(m, cache, i.graph, idgen)
  #  #vm.mode = emRepl
  #  # vm.features = {}
  #  vm.registerAdditionalOps() # Required to register parts of stdlib modules
  #  i.graph.vm = vm
  #  #setPipeLinePass(graph, SemPass)
  #
  #  #setPipeLinePass(graph, SemPass)
  #  setPipeLinePass(i.graph, CgenPass)
  #  i.graph.compilePipelineSystemModule()
  #  i.mainModule = m
  #var m = i.graph.makeModule(file)

  #if first:
  #  # save the ropes for the fields
  #  #discard processPipelineModule(i.graph, i.mainModule, i.idgen, scriptStream)
  #  var backend = cast[BModuleList](i.graph.backend)
  #  for m in cgenModules(backend):
  #    ropesTable[m.filename.string] = default(array[TCFileSection, Rope])
  #    for k in TCFileSection:
  #      ropesTable[m.filename.string][k] = m.s[k]
  #
  #    #if m.filename.string.endsWith("system.nim"): # == "/t/tmp_file.nim":
  #    #  for k in TCFileSection:
  #    #    ropesArray[k] = m.s[k]
  #  #  for x in mitems(m.s):
  #  #    x = Rope""
  #  first = false
  #else:
  #  # reset the ropes
  #  var backend = cast[BModuleList](i.graph.backend)
  #  for m in cgenModules(backend):
  #    if m.filename.string.endsWith("system.nim"): #string == "/t/tmp_file.nim":
  #      for k in TCFileSection:
  #        #if k == cfsInitProc:
  #        echo k, "=====================\n\n"
  #        echo m.s[k]
  #
  #    for k in TCFileSection:
  #      if k == cfsInitProc: #k != cfsTypes:
  #        m.s[k] = ropesTable[m.filename.string][k]

  ## Reset by fully reconstructing module graph etc.
  ## v-- If you comment this out, it will be fast, but break.
  if not first:
    let conf = setupConfig(file, i.graph.config.searchPaths)
    let graph = setupModuleGraph(conf)
    let (m, idgen) = setupModule(file, graph)
    i.graph = graph
    i.mainModule = m
    i.idgen = idgen
  else:
    first = false
  ## Ideas:
  ## We could test (I did that, but maybe did it wrong) making sure to create a *different*
  ## module for each case. I.e. `setupModule` for a secondary module that we recompile.
  ## Maybe somehow like that we can make something work.
  ## Alternative:
  ## Have one ModuleGraph that only does the `SemPass`. Then have a second one that also does
  ## the `CGenPass`. Let the `SemPass` one process the new code. Deep copy data from the SemPass
  ## one to the `CGenPass` version and generate C code. That way the CGenPass one only ever
  ## produces C code once? Huge hack, but could maybe work?
  ## I mean maybe we can generally separate these things without any hacks, but I doubt.

  discard processPipelineModule(i.graph, i.mainModule, i.idgen, scriptStream)
  ## This resetting can also be done by `resetForBackend` I think
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
  initStrTables(i.graph, i.mainModule)
  i.graph.cacheSeqs.clear()
  i.graph.cacheCounters.clear()
  i.graph.cacheTables.clear()
  i.mainModule.ast = nil

  i.graph.config.outDir = AbsoluteDir(file.parentDir)
  i.graph.config.outFile = RelativeFile("lib" & file.extractFilename.replace(".nim", ".so"))
  i.graph.config.projectPath = AbsoluteDir(file.parentDir)
  i.graph.config.projectName = file.extractFilename
  i.graph.config.projectFull = AbsoluteFile file
  echo "File : ", file
  echo i.graph.config.outFile
  echo i.graph.config.projectPath
  echo "--------------------------"

  commandCompileToC(i.graph)

import compiler/passes
proc createInterpreter*(scriptName: string;
                        searchPaths: openArray[string];
                        flags: TSandboxFlags = {},
                        defines = @[("nimscript", "false")],
                        registerOps = true): Interpreter =
  let conf = setupConfig(scriptName, searchPaths)
  let graph = setupModuleGraph(conf)
  let (m, idgen) = setupModule(scriptName, graph)
  result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName, idgen: idgen)

proc destroyInterpreter*(i: Interpreter) =
  ## destructor.
  discard "currently nothing to do."

proc registerErrorHook*(i: Interpreter, hook:
                        proc (config: ConfigRef; info: TLineInfo; msg: string;
                              severity: Severity) {.gcsafe.}) =
  i.graph.config.structuredErrorHook = hook

proc runRepl*(r: TLLRepl;
              searchPaths: openArray[string];
              supportNimscript: bool) =
  ## deadcode but please don't remove... might be revived
  var conf = newConfigRef()
  var cache = newIdentCache()
  var graph = newModuleGraph(cache, conf)

  for p in searchPaths:
    conf.searchPaths.add(AbsoluteDir p)
    if conf.libpath.isEmpty: conf.libpath = AbsoluteDir p

  conf.cmd = cmdInteractive # see also `setCmd`
  conf.setErrorMaxHighMaybe
  initDefines(conf.symbols)
  defineSymbol(conf.symbols, "nimscript")
  if supportNimscript: defineSymbol(conf.symbols, "nimconfig")
  when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
  var m = graph.makeStdinModule()
  incl(m.flags, sfMainModule)
  var idgen = idGeneratorFromModule(m)

  if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph, idgen)
  setPipeLinePass(graph, InterpreterPass)
  graph.compilePipelineSystemModule()
  discard processPipelineModule(graph, m, idgen, llStreamOpenStdIn(r))