Skip to content

Instantly share code, notes, and snippets.

@chgeuer

chgeuer/l.md

Created June 7, 2024 14:16
Show Gist options
  • Save chgeuer/f280e82c5fc418cdf576ae94710076f5 to your computer and use it in GitHub Desktop.
Save chgeuer/f280e82c5fc418cdf576ae94710076f5 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
l.md

Livebook on WSL2

Compile Erlang and Elixir from scratch

Install Linux dependencies

  • autoconf
  • automake
  • build-essential
  • gcc
  • make
  • libnvidia-compute-550
  • libcudnn8
  • cudnn-local-repo-ubuntu2204-8.9.7.29
  • #cudnn-local-repo-ubuntu2204-9.1.1
  • #cudnn9-cuda-12
  • fop
  • xsltproc
  • grep
  • openssl
  • libssl-dev
  • libncurses5-dev
  • libncursesw5-dev
  • libwxgtk-webview3.0-gtk3-dev
  • libwxgtk3.0-gtk3-dev
  • libxml2-utils
  • musl-dev
  • authbind
  • inotify-tools
#!/bin/bash

# Remove stuff
sudo apt -y purge elixir erlang-asn1 erlang-ftp erlang-parsetools erlang-public-key erlang-ssl erlang-tftp erlang-base erlang-base-hipe inotify-tools

sudo apt -y install build-essential automake autoconf libncurses5-dev libncursesw5-dev libssl-dev xsltproc fop libxml2-utils pkg-config libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev inotify-tools

Fetch Erlang and Elixir

Build Erlang

#!/bin/bash


https://github.com/erlang/otp/releases/download/OTP-26.2.5/otp_src_26.2.5.tar.gz

OTP_VERSION="26.2.5" # OTP_VERSION="27.0" 
OTP_DOWNLOAD_URL="https://github.com/erlang/otp/archive/OTP-${OTP_VERSION}.tar.gz"
curl --fail --silent --show-error --location --output "otp-OTP-${OTP_VERSION}.tar.gz" --url "$OTP_DOWNLOAD_URL"
tar xvfz "otp-OTP-${OTP_VERSION}.tar.gz"
cd "otp-OTP-${OTP_VERSION}"
export ERL_TOP=`pwd`
./configure --with-ssl --enable-m64-build
make -j$(nproc)
sudo make install
cd ..
#rm -rf "otp-OTP-${OTP_VERSION}.tar.gz" "otp-OTP-${OTP_VERSION}"

ELIXIR_VERSION="1.16.3" #ELIXIR_VERSION="1.17.0-rc.1"
ELIXIR_DOWNLOAD_URL="https://github.com/elixir-lang/elixir/archive/refs/tags/v${ELIXIR_VERSION}.zip"
curl --fail --silent --show-error --location --output "elixir-${ELIXIR_VERSION}.zip" --url "${ELIXIR_DOWNLOAD_URL}"
unzip "elixir-${ELIXIR_VERSION}.zip"
cd "elixir-${ELIXIR_VERSION}"
make 
sudo make install
#rm -rf "elixir-${ELIXIR_VERSION}.zip"  "elixir-${ELIXIR_VERSION}" 

mix local.hex --force
#mix local.rebar --force && mix local.hex --force

curl --fail --silent --show-error --location --output "rebar3" --url "https://s3.amazonaws.com/rebar3/rebar3"
chmod +x rebar3
sudo mv rebar3 /usr/local/bin

mix archive.install hex phx_new

Elixir

#!/bin/bash

git clone https://github.com/elixir-lang/elixir.git
cd elixir/
git checkout v1.17.0-rc.1
make
sudo make install
cd ..

Livebook

Networking

Check IP of WSL container using wsl.exe hostname -I (in our case that's 172.28.192.25). Then configure Windows to forward the main Livebook and the IFrame ports (4000 and 4001) down to WSL:

On a Windows host, you can forward an IPv4 port from a specific IP/port to another IP/Port: (netsh interface portproxy add v4tov4 listenport=4000 listenaddress=0.0.0.0 connectport=4000 connectaddress=172.28.192.25) and open the Firewall for incoming traffic (netsh advfirewall firewall add rule name="Allow TCP Port 4000 for Livebook" dir=in action=allow protocol=TCP localport=4000)

netsh interface portproxy add v4tov4 listenport=4000 listenaddress=0.0.0.0 connectport=4000 connectaddress=172.28.192.25
netsh advfirewall firewall add rule name="Allow TCP Port 4000 for Livebook" dir=in action=allow protocol=TCP localport=4000

netsh interface portproxy add v4tov4 listenport=4001 listenaddress=0.0.0.0 connectport=4001 connectaddress=172.28.192.25
netsh advfirewall firewall add rule name="Allow TCP Port 4001 for Livebook (IFrame)" dir=in action=allow protocol=TCP localport=4001

netsh interface portproxy add v4tov4 listenport=8443 listenaddress=0.0.0.0 connectport=8443 connectaddress=172.28.192.25
netsh advfirewall firewall add rule name="Allow TCP Port 8443 for Livebook" dir=in action=allow protocol=TCP localport=8443

netsh interface portproxy add v4tov4 listenport=8080 listenaddress=0.0.0.0 connectport=8080 connectaddress=172.28.192.25
netsh advfirewall firewall add rule name="Allow TCP Port 8080 for Livebook" dir=in action=allow protocol=TCP localport=8080

Then on WSL, run the thing:

#!/bin/bash

mkdir $HOME/livebook 

export LIVEBOOK_IP=172.28.192.25 
export LIVEBOOK_PORT=4000 
export LIVEBOOK_IFRAME_PORT=4001
export LIVEBOOK_PASSWORD=SuperSecret123.-
export LIVEBOOK_DATA_PATH=$HOME/livebook 
export LIVEBOOK_HOME=$HOME/livebook 

$HOME/.mix/escripts/livebook server

Fetch drivers

Toolkit: Installed in /usr/local/cuda-12.5/

Please make sure that

Check what's installed for the graphics card

nvidia-smi

This one tells us that CUDA version 12.4 is installed (which I got from cuda_12.4.1_550.54.15_linux.run)

$ nvidia-smi

+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.76.01              Driver Version: 552.22         CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA GeForce RTX 4090        On  |   00000000:01:00.0 Off |                  Off |
|  0%   30C    P8             28W /  480W |   23477MiB /  24564MiB |      2%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|    0   N/A  N/A        30      G   /Xwayland                                   N/A      |
|    0   N/A  N/A      2109      C   /beam.smp                                   N/A      |
+-----------------------------------------------------------------------------------------+

nvcc --version

Also pointing to CUDA 12.4

$ nvcc --version

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2024 NVIDIA Corporation
Built on Thu_Mar_28_02:18:24_PDT_2024
Cuda compilation tools, release 12.4, V12.4.131
Build cuda_12.4.r12.4/compiler.34097967_0

apt-cache policy libcudnn8 | head -n 3

You need libcudnn, in the same version that Nx is compiled against.

This tells us that libcudnn version 8.9 (the one XLA uses) is installed. This came via cudnn-local-repo-ubuntu2204-8.9.7.29_1.0-1_amd64.deb.

Note: Do not install version 9.x (like cudnn-local-repo-ubuntu2204-9.1.1_1.0-1_amd64.deb), that's not (yet) what XLA uses

$ apt-cache policy libcudnn8 | head -n 3

libcudnn8:
  Installed: 8.9.7.29-1+cuda12.2
  Candidate: 8.9.7.29-1+cuda12.2

Do not install version 9 (cudnn-local-repo-ubuntu2204-9.1.1_1.0-1_amd64.deb), but the v8 version

# sudo apt-get -y install cudnn-cuda-12
# sudo apt-get -y install libcudnn8

sudo dpkg -i cudnn-local-repo-ubuntu2204-8.9.7.29_1.0-1_amd64.deb
sudo cp /var/cudnn-local-repo-ubuntu2204-8.9.7.29/cudnn-local-08A7D361-keyring.gpg /usr/share/keyrings/
sudo apt update -y
sudo apt upgrade -y 
sudo apt install -y libcudnn8 libcudnn8-dev nvidia-cuda-toolkig

When using Python, you can also check the GPU via Tensorflor

The last line ([PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]) is the success message. In case of wrong installation, that would just be an empty list ([]).

$ python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"

I tensorflow/core/util/port.cc:113] 
     oneDNN custom operations are on. 
     You may see slightly different numerical results due to floating-point round-off errors from different computation orders.
     To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.

I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:984] 
     could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node
     Your kernel may have been built without NUMA support.

[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]

Livebook config

config/config.exs

config :livebook, LivebookWeb.Endpoint,
  adapter: Bandit.PhoenixAdapter,
  url: [host: "beast.geuer-pollmann.de", path: "/"],
  ...

config/prod.exs

config :livebook, LivebookWeb.Endpoint,
  http: [
    ip: :any, #{127, 0, 0, 1},
    port: 8080,
    http_1_options: [max_header_length: 32768],
    http_2_options: [max_header_value_length: 32768]
  ],
  https: [
    ip: :any, 
    port: 8443, 
    cipher_suite: :strong,
    certfile: "/home/chgeuer/beast.geuer-pollmann.de.crt",
    keyfile: "/home/chgeuer/beast.geuer-pollmann.de.key",
    thousand_island_options: [
      transport_options: [
        sni_fun: &LivebookWeb.Certificates.sni_fun/1
        # socket_opts: [log_level: :info]
      ]
    ]
  ],
  server: true

lib/livebook_web/certificates.ex

defmodule LivebookWeb.Certificates do
  def cert_dir() do
    case :os.type() do
      {:unix, :linux} -> "/home/chgeuer"
      {:win32, :nt} -> "C:/Users/chgeuer/.lego/certificates"
    end
  end

  def sni_fun(host),
    do: [
      cert:
        host
        |> to_string
        |> (fn h -> "#{cert_dir()}/#{h}.crt" end).()
        |> File.read!()
        |> X509.Certificate.from_pem!()
        |> X509.Certificate.to_der(),
        #|> (fn der ->  File.write!("#{cert_dir()}/#{host |> to_string()}.der__", der); der end).(),
      key:
        host
        |> to_string
        |> (fn h -> "#{cert_dir()}/#{h}.key" end).()
        |> File.read!()
        |> X509.PrivateKey.from_pem!()
        |> X509.PrivateKey.to_der(wrap: true)
        |> (fn k -> {:PrivateKeyInfo, k} end).()
    ]
end

mix.exs

defp deps do
    [
      ...
      {:ssl_verify_fun, "~> 1.1.7", manager: :rebar3, override: true},
      {:x509, "~> 0.8.7"},

Get the necessary X.509 Certs

#!/bin/bash

export DNSIMPLE_OAUTH_TOKEN="$(cat ~/.dnsimple_account_token)"

lego \
    --accept-tos=true \
    --dns=dnsimple \
    --email=christian@geuer-pollmann.de \
    --domains=beast.geuer-pollmann.de run
    
cp ~/.lego/certificates/beast.geuer-pollmann.de.crt ~
cp ~/.lego/certificates/beast.geuer-pollmann.de.key ~

Run it

#!/bin/bash

LIVEBOOK_PASSWORD=SuperSecret123.- MIX_ENV=prod mix phx.server MIX_ENV=prod authbind --deep iex --name phoenix@beast.geuer-pollmann.de --cookie "a" -S mix phx.server
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment