Skip to content

Instantly share code, notes, and snippets.

@dacr

dacr/djl-hugging-face-mistral.sc

Last active September 1, 2024 10:15
Download ZIP

Revisions

  1. dacr revised this gist Sep 1, 2024. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -40,6 +40,7 @@ import scala.util.chaining.*

    import scala.io.AnsiColor.{BLUE, BOLD, CYAN, GREEN, MAGENTA, RED, RESET, UNDERLINED, YELLOW}

    // https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF
    val name = "LLM"
    val modelId = "TheBloke/Mistral-7B-Instruct-v0.2-GGUF"
    val quantMethod = "Q4_K_M"
  2. dacr revised this gist Jun 15, 2024. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -17,6 +17,7 @@
    //> using dep "ai.djl:basicdataset:0.28.0"
    //> using dep "ai.djl.llama:llama:0.28.0"
    //> using dep "ai.djl.pytorch:pytorch-engine:0.28.0"
    //> using objectWrapper
    // ---------------------

    /* Thank Scala.IO and NuMind and of course DJL !
  3. dacr revised this gist May 25, 2024. 1 changed file with 7 additions and 7 deletions.
    14 changes: 7 additions & 7 deletions djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -9,14 +9,14 @@
    // run-with : scala-cli $file

    // ---------------------
    //> using scala "3.4.2"
    //> using dep "org.slf4j:slf4j-api:2.0.12"
    //> using dep "org.slf4j:slf4j-simple:2.0.12"
    //> using scala "3.4.2"
    //> using dep "org.slf4j:slf4j-api:2.0.13"
    //> using dep "org.slf4j:slf4j-simple:2.0.13"
    //> using dep "net.java.dev.jna:jna:5.14.0"
    //> using dep "ai.djl:api:0.26.0"
    //> using dep "ai.djl:basicdataset:0.26.0"
    //> using dep "ai.djl.llama:llama:0.26.0"
    //> using dep "ai.djl.pytorch:pytorch-engine:0.26.0"
    //> using dep "ai.djl:api:0.28.0"
    //> using dep "ai.djl:basicdataset:0.28.0"
    //> using dep "ai.djl.llama:llama:0.28.0"
    //> using dep "ai.djl.pytorch:pytorch-engine:0.28.0"
    // ---------------------

    /* Thank Scala.IO and NuMind and of course DJL !
  4. dacr revised this gist May 25, 2024. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -9,7 +9,7 @@
    // run-with : scala-cli $file

    // ---------------------
    //> using scala "3.3.1"
    //> using scala "3.4.2"
    //> using dep "org.slf4j:slf4j-api:2.0.12"
    //> using dep "org.slf4j:slf4j-simple:2.0.12"
    //> using dep "net.java.dev.jna:jna:5.14.0"
  5. dacr revised this gist Feb 18, 2024. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,5 @@
    // summary : using mistral with DJL
    // keywords : djl, machine-learning, mixtral, ai, @testable
    // keywords : djl, machine-learning, llm, mistral, ai, @testable
    // publish : gist
    // authors : David Crosson
    // license : Apache NON-AI License Version 2.0 (https://raw.githubusercontent.com/non-ai-licenses/non-ai-licenses/main/NON-AI-APACHE2)
    @@ -77,13 +77,13 @@ def interact(nextInput: String): String = {
    print(s"${BLUE}$morePrompt$RESET")
    prompt.append(morePrompt)
    in.setInputs(prompt.toString())
    val it = predictor.predict(in)
    val tokens = it.asScala.map(_.getText.tap(print)).toList
    val it = predictor.predict(in)
    val tokens = it.asScala.map(_.getText.tap(print)).toList
    prompt.append(tokens.mkString)
    tokens.mkString
    }

    interact("What is a monad ?")
    interact("Could you give me a scala example ?")
    interact("Thank you very much teacher !")
    println()
    println()
  6. dacr created this gist Feb 17, 2024.
    89 changes: 89 additions & 0 deletions djl-hugging-face-mistral.sc
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,89 @@
    // summary : using mistral with DJL
    // keywords : djl, machine-learning, mixtral, ai, @testable
    // publish : gist
    // authors : David Crosson
    // license : Apache NON-AI License Version 2.0 (https://raw.githubusercontent.com/non-ai-licenses/non-ai-licenses/main/NON-AI-APACHE2)
    // id : 389e67ca-de9a-4f47-a1c0-504564fb2dbe
    // created-on : 2024-02-03T14:34:48+01:00
    // managed-by : https://github.com/dacr/code-examples-manager
    // run-with : scala-cli $file

    // ---------------------
    //> using scala "3.3.1"
    //> using dep "org.slf4j:slf4j-api:2.0.12"
    //> using dep "org.slf4j:slf4j-simple:2.0.12"
    //> using dep "net.java.dev.jna:jna:5.14.0"
    //> using dep "ai.djl:api:0.26.0"
    //> using dep "ai.djl:basicdataset:0.26.0"
    //> using dep "ai.djl.llama:llama:0.26.0"
    //> using dep "ai.djl.pytorch:pytorch-engine:0.26.0"
    // ---------------------

    /* Thank Scala.IO and NuMind and of course DJL !
    https://github.com/numind-tech/scalaio_2024/blob/main/src/main/scala/chatbot/Chatbot.scala
    ------
    djl://ai.djl.huggingface.gguf/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/0.0.1/Q4_K_M, ai.djl.huggingface.gguf/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/0.0.1/Q4_K_M
    https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf?download=true
    */

    System.setProperty("org.slf4j.simpleLogger.defaultLogLevel", "error")

    import ai.djl.repository.zoo.Criteria
    import ai.djl.training.util.ProgressBar
    import ai.djl.llama.engine.LlamaInput
    import ai.djl.llama.engine.LlamaTranslatorFactory
    import ai.djl.llama.jni.Token
    import ai.djl.llama.jni.TokenIterator
    import scala.jdk.CollectionConverters.*
    import scala.util.chaining.*

    import scala.io.AnsiColor.{BLUE, BOLD, CYAN, GREEN, MAGENTA, RED, RESET, UNDERLINED, YELLOW}

    val name = "LLM"
    val modelId = "TheBloke/Mistral-7B-Instruct-v0.2-GGUF"
    val quantMethod = "Q4_K_M"
    val url = s"djl://ai.djl.huggingface.gguf/$modelId/0.0.1/$quantMethod"

    val criteria =
    Criteria.builder
    .setTypes(classOf[LlamaInput], classOf[TokenIterator])
    .optModelUrls(url)
    .optOption("number_gpu_layers", "43")
    .optTranslatorFactory(new LlamaTranslatorFactory())
    .optProgress(new ProgressBar)
    .build

    val model = criteria.loadModel()
    val predictor = model.newPredictor()

    val param = new LlamaInput.Parameters()
    param.setTemperature(0.7f)
    param.setPenalizeNl(true)
    param.setMirostat(2)
    param.setAntiPrompt(Array("User: "))

    val in = new LlamaInput()
    in.setParameters(param)

    val systemPrompt =
    s"""As a computer science teacher, I make my best to help my students to become software experts.
    |
    |$name: How may I help you today ?""".stripMargin

    val prompt = StringBuilder(systemPrompt)

    def interact(nextInput: String): String = {
    val morePrompt = s"\nUser: $nextInput\n$name: "
    print(s"${BLUE}$morePrompt$RESET")
    prompt.append(morePrompt)
    in.setInputs(prompt.toString())
    val it = predictor.predict(in)
    val tokens = it.asScala.map(_.getText.tap(print)).toList
    prompt.append(tokens.mkString)
    tokens.mkString
    }

    interact("What is a monad ?")
    interact("Could you give me a scala example ?")
    interact("Thank you very much teacher !")
    println()