Skip to content

Instantly share code, notes, and snippets.

@jonatanklosko
Last active July 12, 2024 16:54
Show Gist options
  • Save jonatanklosko/ec56a3a5d0520624903e9f75ebd7b171 to your computer and use it in GitHub Desktop.
Save jonatanklosko/ec56a3a5d0520624903e9f75ebd7b171 to your computer and use it in GitHub Desktop.
# Usage: mix run livebook_run.exs [.livemd file]
defmodule LivebookRun.Client do
use GenServer
alias Livebook.{Session, LiveMarkdown}
def run_and_save(notebook_path) do
{:ok, pid} = GenServer.start(__MODULE__, {notebook_path})
ref = Process.monitor(pid)
receive do
{:DOWN, ^ref, :process, _pid, _reason} -> :ok
end
end
@impl true
def init({notebook_path}) do
{notebook, _messages} =
notebook_path
|> File.read!()
|> LiveMarkdown.notebook_from_livemd()
{:ok, session} = Livebook.Sessions.create_session(notebook: notebook)
{data, _client_id} = Session.register_client(session.pid, self(), Livebook.Users.User.new())
Session.subscribe(session.id)
Session.queue_full_evaluation(session.pid, [])
{:ok, %{notebook_path: notebook_path, session: session, data: data, evaluated_count: 0}}
end
@impl true
def handle_info({:operation, operation}, state) do
data =
case Session.Data.apply_operation(state.data, operation) do
{:ok, data, _actions} -> data
:error -> state.data
end
{evaluated_count, evaluable_count} = evaluation_progress(data)
if evaluated_count != state.evaluated_count do
IO.puts("Evaluated: #{evaluated_count}/#{evaluable_count}")
end
if evaluated_count == evaluable_count do
{content, _} = LiveMarkdown.notebook_to_livemd(data.notebook, include_outputs: true)
File.write!(state.notebook_path, content)
{:stop, :shutdown, state}
else
{:noreply, %{state | data: data, evaluated_count: evaluated_count}}
end
end
def handle_info(_message, state), do: {:noreply, state}
defp evaluation_progress(data) do
evaluable = Livebook.Notebook.evaluable_cells_with_section(data.notebook)
evaluated_count =
Enum.count(evaluable, fn {cell, _} ->
match?(%{validity: :evaluated, status: :ready}, data.cell_infos[cell.id].eval)
end)
{evaluated_count, length(evaluable)}
end
end
defmodule LivebookRun.App do
def main() do
case System.argv() do
[notebook_path] ->
LivebookRun.Client.run_and_save(notebook_path)
IO.puts("Saved notebook with output to #{notebook_path}")
_args ->
IO.write("""
Usage: mix run #{Path.relative_to_cwd(__ENV__.file)} [.livemd file]
Runs the given notebook and saves it into the same file including outputs.
""")
end
end
end
LivebookRun.App.main()
@joshuataylor
Copy link

joshuataylor commented Jul 12, 2024

This is awesome, thanks!

Unfortunately, it seems to be giving this error, as of 2024-07-12 (v0.13.3):

[error] GenServer #PID<0.375.0> terminating
** (File.Error) could not write to file "/Users/josh/dev/lb/test1.livemd": bad argument
    (elixir 1.17.2) lib/file.ex:1144: File.write!/3
    livebook_run.exs:51: LivebookRun.Client.handle_info/2
    (stdlib 6.0.1) gen_server.erl:2173: :gen_server.try_handle_info/3
    (stdlib 6.0.1) gen_server.erl:2261: :gen_server.handle_msg/6
    (stdlib 6.0.1) proc_lib.erl:329: :proc_lib.init_p_do_apply/3
Last message: {:operation, {:add_cell_evaluation_response, "__server__", "3bx7r2u4x3tq6j6n", %{type: :terminal_text, text: "\e[35mnil\e[0m", chunk: false}, %{interrupted: false, errored: false, identifiers_defined: %{}, identifiers_used: [:pdict], evaluation_time_ms: 0, code_markers: []}}}
State: %{data: %Livebook.Session.Data{notebook: %Livebook.Notebook{name: "Untitled notebook", setup_section: %Livebook.Notebook.Section{id: "setup-section", name: "Setup", cells: [%Livebook.Notebook.Cell.Code{id: "setup", source: "Mix.install([\n  {:kino, \"~> 0.13.2\"}  \n])", outputs: [{0, %{type: :terminal_text, text: "\e[34m:ok\e[0m", chunk: false}}], language: :elixir, reevaluate_automatically: false, continue_on_error: false}], parent_id: nil}, sections: [%Livebook.Notebook.Section{id: "pqglriyz7jqcvtec", name: "Section", cells: [%Livebook.Notebook.Cell.Code{id: "3bx7r2u4x3tq6j6n", source: "", outputs: [], language: :elixir, reevaluate_automatically: false, continue_on_error: false}], parent_id: nil}], leading_comments: [], persist_outputs: false, autosave_interval_s: 5, default_language: :elixir, output_counter: 1, app_settings: %Livebook.Notebook.AppSettings{slug: nil, multi_session: false, zero_downtime: false, show_existing_sessions: false, auto_shutdown_ms: nil, access_type: :protected, password: "kisll7zhsb45weov", show_source: false, output_type: :all}, hub_id: "personal-hub", hub_secret_names: [], file_entries: [], quarantine_file_entry_names: MapSet.new([]), teams_enabled: false, deployment_group_id: nil}, origin: nil, file: nil, dirty: true, persistence_warnings: [], section_infos: %{"pqglriyz7jqcvtec" => %{evaluating_cell_id: "3bx7r2u4x3tq6j6n", evaluation_queue: MapSet.new([])}, "setup-section" => %{evaluating_cell_id: nil, evaluation_queue: MapSet.new([])}}, cell_infos: %{"3bx7r2u4x3tq6j6n" => %{eval: %{data: %Livebook.Session.Data{notebook: %Livebook.Notebook{name: "Untitled notebook", setup_section: %Livebook.Notebook.Section{id: "setup-section", name: "Setup", cells: [%Livebook.Notebook.Cell.Code{id: "setup", source: "Mix.install([\n  {:kino, \"~> 0.13.2\"}  \n])", outputs: [{0, %{type: :terminal_text, text: "\e[34m:ok\e[0m", chunk: false}}], language: :elixir, reevaluate_automatically: false, continue_on_error: false}], parent_id: nil}, sections: [%Livebook.Notebook.Section{id: "pqglriyz7jqcvtec", name: "Section", cells: [%Livebook.Notebook.Cell.Code{id: "3bx7r2u4x3tq6j6n", source: "", outputs: [], language: :elixir, reevaluate_automatically: false, continue_on_error: false}], parent_id: nil}], leading_comments: [], persist_outputs: false, autosave_interval_s: 5, default_language: :elixir, output_counter: 1, app_settings: %Livebook.Notebook.AppSettings{slug: nil, multi_session: false, zero_downtime: false, show_existing_sessions: false, auto_shutdown_ms: nil, access_type: :protected, password: "kisll7zhsb45weov", show_source: false, output_type: :all}, hub_id: "personal-hub", hub_secret_names: [], file_entries: [], quarantine_file_entry_names: MapSet.new([]), teams_enabled: false, deployment_group_id: nil}, origin: nil, file: nil, dirty: true, persistence_warnings: [], section_infos: %{"pqglriyz7jqcvtec" => %{evaluating_cell_id: nil, evaluation_queue: MapSet.new(["3bx7r2u4x3tq6j6n"])}, "setup-section" => %{evaluating_cell_id: nil, evaluation_queue: MapSet.new([])}}, cell_infos: %{"3bx7r2u4x3tq6j6n" => %{eval: %{data: nil, status: :queued, interrupted: false, validity: :fresh, new_bound_to_inputs: %{}, evaluation_digest: nil, bound_to_inputs: %{}, errored: false, identifiers_defined: %{}, identifiers_used: [], snapshot: 120034072, evaluation_snapshot: nil, reevaluates_automatically: false, evaluation_time_ms: nil, code_markers: [], evaluation_end: nil, evaluation_number: 0, outputs_batch_number: 0, doctest_reports: %{}, evaluation_start: nil}, sources: %{primary: %{revision: 0, digest: <<212, 29, 140, 217, 143, 0, 178, 4, 233, 128, 9, 152, 236, 248, 66, 126>>, deltas: [], revision_by_client_id: %{"57k2pbrvtu5vi7bp" => 0}}}}, "setup" => %{eval: %{data: nil, status: :ready, interrupted: false, validity: :evaluated, new_bound_to_inputs: %{}, evaluation_digest: <<103, 102, 193, 30, 22, 240, 226, 91, 182, 34, 105, 169, 207, 212, 97, 178>>, bound_to_inputs: %{}, errored: false, identifiers_defined: %{}, identifiers_used: [:pdict, {:alias, Mix}, {:module, Mix}], snapshot: 120034072, evaluation_snapshot: 120034072, reevaluates_automatically: false, evaluation_time_ms: 71, code_markers: [], evaluation_end: ~U[2024-07-12 12:49:59.550346Z], evaluation_number: 1, outputs_batch_number: 1, doctest_reports: %{}, evaluation_start: ~U[2024-07-12 12:49:59.470107Z]}, sources: %{primary: %{revision: 0, digest: <<103, 102, 193, 30, 22, 240, 226, 91, 182, 34, 105, 169, 207, 212, 97, 178>>, deltas: [], revision_by_client_id: %{"57k2pbrvtu5vi7bp" => 0}}}}}, input_infos: %{}, bin_entries: [], runtime: %Livebook.Runtime.ElixirStandalone{node: :"livebook_kpdnktd6--udl56lti@127.0.0.1", server_pid: #PID<31654.137.0>}, runtime_transient_state: %{}, runtime_connected_nodes: [], smart_cell_definitions: [%{name: "Database connection", kind: "Elixir.KinoDB.ConnectionCell", requirement_presets: [%{name: "Amazon Athena", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "req_athena", dependency: %{config: [], dep: {:req_athena, ">= 0.0.0"}}}]}, %{name: "Google BigQuery", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "req_bigquery", dependency: %{config: [], dep: {:req_bigquery, ">= 0.0.0"}}}]}, %{name: "MySQL", packages: [%{name: "kino_db", dependency: %{config: [], dep:{:kino_db, "~> 0.2.8"}}}, %{name: "myxql", dependency: %{config: [], dep: {:myxql, ">= 0.0.0"}}}]}, %{name: "PostgreSQL", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "postgrex", dependency: %{config: [], dep: {:postgrex, ">= 0.0.0"}}}]}, %{name: "Snowflake", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "kino_explorer", dependency: %{config: [], dep: {:kino_explorer, "~> 0.1.20"}}}, %{name: "adbc", dependency: %{config: [], dep: {:adbc, ">= 0.0.0"}}}]}, %{name: "SQLite", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "exqlite", dependency: %{config: [], dep: {:exqlite, "~> 0.23.0"}}}]}, %{name: "SQLServer", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}, %{name: "tds", dependency: %{config: [], dep: {:tds, ">= 0.0.0"}}}]}]}, %{name: "SQL query", kind: "Elixir.KinoDB.SQLCell", requirement_presets: [%{name: "Default", packages: [%{name: "kino_db", dependency: %{config: [], dep: {:kino_db, "~> 0.2.8"}}}]}]}, %{name: "Chart", kind: "Elixir.KinoVegaLite.ChartCell", requirement_presets: [%{name: "Default", packages: [%{name: "kino_vega_lite", dependency: %{config: [], dep: {:kino_vega_lite, "~> 0.1.11"}}}]}]}, %{name: "Map", kind: "Elixir.KinoMapLibre.MapCell", requirement_presets: [%{name: "Default", packages: [%{name: "kino_maplibre", dependency: %{config: [], dep: {:kino_maplibre, "~> 0.1.12"}}}]}]}, %{name: "Slack message", kind: "Elixir.KinoSlack.MessageCell", requirement_presets: [%{name: "Default", packages: [%{name: "kino_slack", dependency: %{config: [], dep: {:kino_slack, "~> 0.1.1"}}}]}]}, %{name: "Neural Network task", kind: "Elixir.KinoBumblebee.TaskCell", requirement_presets: [%{name: "Default", packages: [%{name: "kino_bumblebee", dependency: %{config: [], dep: {:kino_bumblebee, "~> 0.5.0"}}}, %{name: "exla", dependency: %{config: [nx: [default_backend: EXLA.Backend]], dep: {:exla, ">= 0.0.0"}} (truncated)

File:
https://gist.github.com/joshuataylor/8134b502296228d23390d850174cf14e

@jonatanklosko
Copy link
Author

jonatanklosko commented Jul 12, 2024

@joshuataylor oh that makes sense, this script was 2 years old! Should be fixed now :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment