Skip to content

Instantly share code, notes, and snippets.

@slashmili

slashmili/langchain-bumblebee.livemd Secret

Created March 23, 2024 14:05
Show Gist options
  • Save slashmili/ba0ac06a6346e793e357caf940a8a424 to your computer and use it in GitHub Desktop.
Save slashmili/ba0ac06a6346e793e357caf940a8a424 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
langchain-bumblebee.livemd

Bumbelbee/Llama/Langchain

Mix.install([
  {:bumblebee, "~> 0.5.3"},
  {:langchain, "~> 0.1.10"},
  {:nx, "~> 0.7.1"},
  {:exla, "~> 0.7.1"},
  {:kino, "~> 0.12.3"}
])

Nx.global_default_backend(EXLA.Backend)

Section

repo =
  {:hf, "meta-llama/Llama-2-7b-chat-hf", auth_token: System.fetch_env!("LB_HUGGINGFACE_TOKEN")}

{:ok, model_info} = Bumblebee.load_model(repo, type: :bf16)
{:ok, tokenizer} = Bumblebee.load_tokenizer(repo)
{:ok, generation_config} = Bumblebee.load_generation_config(repo)

generation_config =
  Bumblebee.configure(generation_config,
    max_new_tokens: 256,
    strategy: %{type: :multinomial_sampling, top_p: 0.6}
  )

13:32:42.064 [info] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355

13:32:42.064 [info] XLA service 0x7fb47c02ab60 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:

13:32:42.064 [info]   StreamExecutor device (0): NVIDIA A100-SXM4-80GB, Compute Capability 8.0

13:32:42.065 [info] Using BFC allocator.

13:32:42.065 [info] XLA backend allocating 76488022425 bytes on device 0 for BFCAllocator.

13:32:43.915 [info] Loaded cuDNN version 8907

13:32:43.921 [info] Start cannot spawn child process: No such file or directory

13:32:43.926 [info] Start cannot spawn child process: No such file or directory

13:32:43.931 [info] Using nvlink for parallel linking
|===============================================================| 100% (1.84 MB)

%Bumblebee.Text.GenerationConfig{
  max_new_tokens: 256,
  min_new_tokens: nil,
  max_length: nil,
  min_length: nil,
  strategy: %{type: :multinomial_sampling, top_p: 0.6},
  decoder_start_token_id: nil,
  forced_bos_token_id: nil,
  forced_eos_token_id: nil,
  forced_token_ids: [],
  suppressed_token_ids: [],
  no_repeat_ngram_length: nil,
  temperature: nil,
  bos_token_id: 1,
  eos_token_id: 2,
  pad_token_id: 0,
  extra_config: nil
}

serving =
  Bumblebee.Text.generation(model_info, tokenizer, generation_config,
    compile: [batch_size: 1, sequence_length: 1028],
    defn_options: [compiler: EXLA, lazy_transfers: :always],
    stream: true,
    stream_done: true
  )

Kino.start_child({Nx.Serving, name: Llama2Serving, serving: serving})

13:33:49.600 [info] ptxas warning : Registers are spilled to local memory in function 'triton_gemm_dot_515', 4 bytes spill stores, 4 bytes spill loads


13:33:49.730 [info] ptxas warning : Registers are spilled to local memory in function 'triton_gemm_dot_515', 104 bytes spill stores, 152 bytes spill loads


13:34:01.545 [info] ptxas warning : Registers are spilled to local memory in function 'sort_7531', 648 bytes spill stores, 632 bytes spill loads


13:34:01.559 [info] ptxas warning : Registers are spilled to local memory in function 'sort_3', 648 bytes spill stores, 632 bytes spill loads



{:ok, #PID<0.350.0>}

chat =
  LangChain.ChatModels.ChatBumblebee.new!(%{
    serving: Llama2Serving,
    template_format: :llama_2,
    stream: true
  })
%LangChain.ChatModels.ChatBumblebee{
  serving: Llama2Serving,
  template_format: :llama_2,
  stream: true,
  seed: nil
}

alias LangChain.Function
alias LangChain.Message
alias LangChain.Chains.LLMChain
alias LangChain.ChatModels.ChatOpenAI

# map of data we want to be passed as `context` to the function when
# executed.
custom_context = %{
  "user_id" => 123,
  "hairbrush" => "drawer",
  "dog" => "backyard",
  "sandwich" => "kitchen"
}

# a custom Elixir function made available to the LLM
custom_fn =
  Function.new!(%{
    name: "custom",
    description: "Returns the location of the requested element or item.",
    parameters_schema: %{
      type: "object",
      properties: %{
        thing: %{
          type: "string",
          description: "The thing whose location is being requested."
        }
      },
      required: ["thing"]
    },
    function: fn %{"thing" => thing} = _arguments, context ->
      # our context is a pretend item/location location map
      context[thing]
    end
  })

# create and run the chain
{:ok, updated_chain, %Message{} = message} =
  LLMChain.new!(%{
    llm: chat,
    custom_context: custom_context,
    verbose: true
  })
  |> LLMChain.add_functions(custom_fn)
  |> LLMChain.add_message(Message.new_user!("Where is the hairbrush located?"))
  |> LLMChain.run(while_needs_response: true)

# print the LLM's answer
IO.puts(message.content)
# => "The hairbrush is located in the drawer."
LLM: %LangChain.ChatModels.ChatBumblebee{
  serving: Llama2Serving,
  template_format: :llama_2,
  stream: true,
  seed: nil
}
MESSAGES: [
  %LangChain.Message{
    content: "Where is the hairbrush located?",
    index: nil,
    status: :complete,
    role: :user,
    function_name: nil,
    arguments: nil
  }
]
FUNCTIONS: [
  %LangChain.Function{
    name: "custom",
    description: "Returns the location of the requested element or item.",
    display_text: nil,
    function: #Function<41.105768164/2 in :erl_eval.expr/6>,
    parameters_schema: %{
      type: "object",
      required: ["thing"],
      properties: %{
        thing: %{
          type: "string",
          description: "The thing whose location is being requested."
        }
      }
    },
    parameters: []
  }
]

13:55:41.589 [warning] Streaming call requested but no callback function was given.

13:55:41.603 [warning] Streaming call requested but no callback function was given.

13:55:41.617 [warning] Streaming call requested but no callback function was given.

13:55:41.658 [warning] Streaming call requested but no callback function was given.

13:55:41.671 [warning] Streaming call requested but no callback function was given.

13:55:41.713 [warning] Streaming call requested but no callback function was given.

13:55:41.726 [warning] Streaming call requested but no callback function was given.

13:55:41.740 [warning] Streaming call requested but no callback function was given.

13:55:41.754 [warning] Streaming call requested but no callback function was given.

13:55:41.767 [warning] Streaming call requested but no callback function was given.

13:55:41.781 [warning] Streaming call requested but no callback function was given.

13:55:41.808 [warning] Streaming call requested but no callback function was given.

13:55:41.822 [warning] Streaming call requested but no callback function was given.

13:55:41.836 [warning] Streaming call requested but no callback function was given.

13:55:41.849 [warning] Streaming call requested but no callback function was given.

13:55:41.863 [warning] Streaming call requested but no callback function was given.

13:55:41.876 [warning] Streaming call requested but no callback function was given.

13:55:41.904 [warning] Streaming call requested but no callback function was given.

13:55:41.918 [warning] Streaming call requested but no callback function was given.

13:55:41.932 [warning] Streaming call requested but no callback function was given.

13:55:41.945 [warning] Streaming call requested but no callback function was given.

13:55:41.973 [warning] Streaming call requested but no callback function was given.

13:55:41.986 [warning] Streaming call requested but no callback function was given.

13:55:42.000 [warning] Streaming call requested but no callback function was given.

13:55:42.054 [warning] Streaming call requested but no callback function was given.

13:55:42.081 [warning] Streaming call requested but no callback function was given.

13:55:42.095 [warning] Streaming call requested but no callback function was given.

13:55:42.109 [warning] Streaming call requested but no callback function was given.

13:55:42.122 [warning] Streaming call requested but no callback function was given.

13:55:42.136 [warning] Streaming call requested but no callback function was given.

13:55:42.150 [warning] Streaming call requested but no callback function was given.

13:55:42.164 [warning] Streaming call requested but no callback function was given.

13:55:42.177 [warning] Streaming call requested but no callback function was given.

13:55:42.191 [warning] Streaming call requested but no callback function was given.

13:55:42.204 [warning] Streaming call requested but no callback function was given.

13:55:42.218 [warning] Streaming call requested but no callback function was given.

13:55:42.273 [warning] Streaming call requested but no callback function was given.

13:55:42.287 [warning] Streaming call requested but no callback function was given.

13:55:42.301 [warning] Streaming call requested but no callback function was given.

13:55:42.314 [warning] Streaming call requested but no callback function was given.

13:55:42.328 [warning] Streaming call requested but no callback function was given.

13:55:42.342 [warning] Streaming call requested but no callback function was given.

13:55:42.355 [warning] Streaming call requested but no callback function was given.

13:55:42.383 [warning] Streaming call requested but no callback function was given.

13:55:42.396 [warning] Streaming call requested but no callback function was given.

13:55:42.437 [warning] Streaming call requested but no callback function was given.

13:55:42.451 [warning] Streaming call requested but no callback function was given.

13:55:42.478 [warning] Streaming call requested but no callback function was given.

13:55:42.533 [warning] Streaming call requested but no callback function was given.

13:55:42.546 [warning] Streaming call requested but no callback function was given.

13:55:42.560 [warning] Streaming call requested but no callback function was given.

13:55:42.574 [warning] Streaming call requested but no callback function was given.

13:55:42.587 [warning] Streaming call requested but no callback function was given.

13:55:42.601 [warning] Streaming call requested but no callback function was given.

13:55:42.629 [warning] Streaming call requested but no callback function was given.

13:55:42.642 [warning] Streaming call requested but no callback function was given.

13:55:42.656 [warning] Streaming call requested but no callback function was given.

13:55:42.670 [warning] Streaming call requested but no callback function was given.

13:55:42.684 [warning] Streaming call requested but no callback function was given.

13:55:42.711 [warning] Streaming call requested but no callback function was given.

13:55:42.753 [warning] Streaming call requested but no callback function was given.

13:55:42.767 [warning] Streaming call requested but no callback function was given.

13:55:42.780 [warning] Streaming call requested but no callback function was given.

13:55:42.794 [warning] Streaming call requested but no callback function was given.

13:55:42.808 [warning] Streaming call requested but no callback function was given.

13:55:42.821 [warning] Streaming call requested but no callback function was given.

13:55:42.835 [warning] Streaming call requested but no callback function was given.

13:55:42.890 [warning] Streaming call requested but no callback function was given.

13:55:42.904 [warning] Streaming call requested but no callback function was given.

13:55:42.918 [warning] Streaming call requested but no callback function was given.

13:55:42.945 [warning] Streaming call requested but no callback function was given.

13:55:42.973 [warning] Streaming call requested but no callback function was given.

13:55:42.987 [warning] Streaming call requested but no callback function was given.

13:55:43.000 [warning] Streaming call requested but no callback function was given.

13:55:43.014 [warning] Streaming call requested but no callback function was given.

13:55:43.027 [warning] Streaming call requested but no callback function was given.

13:55:43.041 [warning] Streaming call requested but no callback function was given.

13:55:43.068 [warning] Streaming call requested but no callback function was given.

13:55:43.095 [warning] Streaming call requested but no callback function was given.

13:55:43.109 [warning] Streaming call requested but no callback function was given.

13:55:43.136 [warning] Streaming call requested but no callback function was given.

13:55:43.177 [warning] Streaming call requested but no callback function was given.

13:55:43.190 [warning] Streaming call requested but no callback function was given.

13:55:43.204 [warning] Streaming call requested but no callback function was given.

13:55:43.217 [warning] Streaming call requested but no callback function was given.

13:55:43.231 [warning] Streaming call requested but no callback function was given.

13:55:43.286 [warning] Streaming call requested but no callback function was given.

13:55:43.300 [warning] Streaming call requested but no callback function was given.

13:55:43.314 [warning] Streaming call requested but no callback function was given.

13:55:43.328 [warning] Streaming call requested but no callback function was given.

13:55:43.355 [warning] Streaming call requested but no callback function was given.

13:55:43.369 [warning] Streaming call requested but no callback function was given.

13:55:43.383 [warning] Streaming call requested but no callback function was given.

13:55:43.397 [warning] Streaming call requested but no callback function was given.

13:55:43.424 [warning] Streaming call requested but no callback function was given.

13:55:43.437 [warning] Streaming call requested but no callback function was given.

13:55:43.478 [warning] Streaming call requested but no callback function was given.

13:55:43.492 [warning] Streaming call requested but no callback function was given.

13:55:43.506 [warning] Streaming call requested but no callback function was given.

13:55:43.520 [warning] Streaming call requested but no callback function was given.

13:55:43.533 [warning] Streaming call requested but no callback function was given.

13:55:43.561 [warning] Streaming call requested but no callback function was given.

13:55:43.574 [warning] Streaming call requested but no callback function was given.

13:55:43.588 [warning] Streaming call requested but no callback function was given.

13:55:43.602 [warning] Streaming call requested but no callback function was given.

13:55:43.615 [warning] Streaming call requested but no callback function was given.

13:55:43.629 [warning] Streaming call requested but no callback function was given.

13:55:43.642 [warning] Streaming call requested but no callback function was given.

13:55:43.656 [warning] Streaming call requested but no callback function was given.

13:55:43.670 [warning] Streaming call requested but no callback function was given.

13:55:43.683 [warning] Streaming call requested but no callback function was given.

13:55:43.697 [warning] Streaming call requested but no callback function was given.

13:55:43.739 [warning] Streaming call requested but no callback function was given.

13:55:43.752 [warning] Streaming call requested but no callback function was given.

13:55:43.766 [warning] Streaming call requested but no callback function was given.

13:55:43.780 [warning] Streaming call requested but no callback function was given.

13:55:43.793 [warning] Streaming call requested but no callback function was given.

13:55:43.834 [warning] Streaming call requested but no callback function was given.

13:55:43.848 [warning] Streaming call requested but no callback function was given.

13:55:43.862 [warning] Streaming call requested but no callback function was given.

13:55:43.876 [warning] Streaming call requested but no callback function was given.

13:55:43.889 [warning] Streaming call requested but no callback function was given.

13:55:43.903 [warning] Streaming call requested but no callback function was given.

13:55:43.944 [warning] Streaming call requested but no callback function was given.

13:55:43.944 [warning] Streaming call requested but no callback function was given.
COMBINED DELTA MESSAGE RESPONSE: %LangChain.Message{
  content: "I'm just an AI, I don't have access to your personal belongings or the layout of your home, so I cannot accurately locate your hairbrush. However, I can provide you with some general information about where hairbrushes are typically kept in a typical home.\n\nIn many households, hairbrushes are usually stored in a bathroom cabinet or on a bathroom countertop. Some people may also keep their hairbrushes in a dresser drawer or in a designated hair accessory case.\n\nIf you're having trouble finding your hairbrush, you might want to check these locations first. If you're still unable to find it, you could try asking other members of your household if they've seen it or check underneath your bed or in your closet.",
  index: nil,
  status: :complete,
  role: :assistant,
  function_name: nil,
  arguments: nil
}
I'm just an AI, I don't have access to your personal belongings or the layout of your home, so I cannot accurately locate your hairbrush. However, I can provide you with some general information about where hairbrushes are typically kept in a typical home.

In many households, hairbrushes are usually stored in a bathroom cabinet or on a bathroom countertop. Some people may also keep their hairbrushes in a dresser drawer or in a designated hair accessory case.

If you're having trouble finding your hairbrush, you might want to check these locations first. If you're still unable to find it, you could try asking other members of your household if they've seen it or check underneath your bed or in your closet.

:ok
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment