Skip to content

Instantly share code, notes, and snippets.

@goldfita
Last active May 18, 2024 15:58
Show Gist options
  • Save goldfita/20c1e002b1188b054aa27ea271cd8c8e to your computer and use it in GitHub Desktop.
Save goldfita/20c1e002b1188b054aa27ea271cd8c8e to your computer and use it in GitHub Desktop.
Custom containers for development using Linux namespaces
# Deal with 'Logon failure: the user has not been granted the requested logon type at this computer.'
# This seems to BSOD Windows
#Get-Service vmcompute | Restart-Service
# Add the vEthernet adapter
wsl hostname -I
# Get IPs
$ips = New-Object System.Collections.Generic.HashSet[string]
$portproxy = netsh interface portproxy show v4tov4
foreach ($line in $portproxy)
{
if ($line -match '^(\d.*?)\s.*$') {
$ip = $Matches.1
($ips.Add($ip)) | out-null
}
}
Write-Host $ips
# Add static IP to vEthernet adapter
$adapterName = (Get-NetAdapter -IncludeHidden | Where-Object {$_.Name -match '^vEthernet.*WSL'}).Name
foreach ($ip in $ips) {
Write-Host $ip
netsh.exe interface ip add address "${adapterName}" "${ip}" 255.255.255.0
}
#!/bin/bash
if [ ! -z "$VM_NUM" ]; then
echo "This is already a container. Quitting."
exit 1
fi
if [ "root" != "$USER" ]; then
VM_NUM=$container_num source .bashrc
exec sudo -E PATH="$PATH" "$0" "$@"
fi
cmd="$1"
container_num="$2"
containers_path="/containers"
container_path="${containers_path}/${container_num}"
merge_path="${container_path}/merge"
ns_path="${container_path}/ns"
vnet="vnet${container_num}"
veth_ns="veth${container_num}"
veth_host="br-veth${container_num}"
net_path="/var/run/netns"
vnet_path="${net_path}/${vnet}"
chain_forward="container${container_num}-forward"
chain_preroute="container${container_num}-preroute"
user=goldfita
hostname="$(hostname)"
br="br1"
eth="eth0"
offset=1000
subnet=192.168.200
namespace_dirs_excl_net=(mount pid)
namespace_dirs=("${namespace_dirs_excl_net[@]}" net)
ports_excl_web=(22)
ports_web=(80 443)
ports=("${ports_excl_web[@]}" "${ports_web[@]}")
overlay_paths=(/home/"$user" /var/log)
octet=$container_num
for nsd in "${namespace_dirs_excl_net[@]}"; do
namespace_flags_excl_net="${namespace_flags_excl_net} --${nsd}=${ns_path}/${nsd}";
done
namespace_flags="${namespace_flags_excl_net} --net=${ns_path}/net"
pid_search="unshare.*${container_path}"
switch_user=(sudo -i --preserve-env="VM_NUM,WSL_DISTRO_NAME,WSL_INTEROP,PATH" -u "$user" VM_NUM=$container_num)
if [[ ! "$container_num" =~ [[:digit:]]+ ]]; then
echo "You forgot to specify a container number (${container_num})."
exit 1
fi
# Network namespaces
# https://superuser.com/questions/1715273/wsl2-two-separate-centos-distributions-have-same-eth0-inet-addres/1715457#1715457
# https://www.gilesthomas.com/2021/03/fun-with-network-namespaces
####################################################################
function none_mounted_excl_net {
for nsd in "${namespace_dirs_excl_net[@]}"; do
mountpoint -q "${ns_path}/${nsd}" && return 1
done
return 0
}
function none_mounted {
mountpoint -q "${ns_path}/net" && return 1
none_mounted_excl_net
return $?
}
function unmount_ns_excl_net {
for nsd in "${namespace_dirs_excl_net[@]}"; do
umount "${ns_path}/${nsd}" 2> /dev/null
done
}
function unmount_ns {
unmount_ns_excl_net
umount "$vnet_path" 2> /dev/null
umount "${ns_path}/net" 2> /dev/null
umount "$ns_path" 2> /dev/null
}
function remove_vnet {
rm "$vnet_path" 2> /dev/null
ip link delete "$veth_host" 2> /dev/null
}
function add_bridge {
ip link add name br1 type bridge
ip addr add "${subnet}.0/24" brd + dev "$br"
ip link set "$br" up
}
function add_net {
if [ ! -e /sys/devices/virtual/net/br1 ]; then
ip addr add "192.168.1.0/16" dev "$eth" 2> /dev/null
add_bridge
add_forwarding
fi
}
function add_forwarding {
iptables -P FORWARD DROP
iptables -A FORWARD -o eth0 -i "$br" -j ACCEPT
iptables -A FORWARD -i eth0 -o "$br" -j ACCEPT
iptables -t nat -A POSTROUTING -s "${subnet}.0/255.255.255.0" -o "$eth" -j MASQUERADE
sysctl -w net.ipv4.ip_forward=1 > /dev/null
}
function add_port_forwarding {
dest_ip="${subnet}.${octet}"
# Add/replace chains
iptables -F "$chain_forward" 2> /dev/null
iptables -F "$chain_preroute" -t nat 2> /dev/null
iptables -N "$chain_forward" 2> /dev/null
iptables -N "$chain_preroute" -t nat 2> /dev/null
if [[ ! "$(iptables -S)" =~ $dest_ip ]]; then
iptables -A FORWARD -d "${dest_ip}" -p tcp -m tcp -m state --state NEW,RELATED,ESTABLISHED -j "$chain_forward"
fi
if [[ ! "$(iptables -S -t nat)" =~ ${eth}.*${chain_preroute} ]]; then
iptables -t nat -A PREROUTING -i "$eth" -p tcp -m tcp -j "$chain_preroute"
fi
# These will be handled by nginx
for port in "${ports_excl_web[@]}"; do
new_port=$(($port + $container_num * $offset))
iptables -t nat -A "$chain_preroute" -p tcp -m tcp --dport $new_port -j DNAT --to-destination "${dest_ip}:${port}"
echo "Forwarding from ${subnet}.0:${new_port} to ${dest_ip}:${port}."
done
for port in "${ports[@]}"; do
iptables -A "$chain_forward" -p tcp -m tcp --dport $port -j ACCEPT
done
}
function add_host_net {
ip link add "$veth_ns" type veth peer name "$veth_host"
ip link set "$veth_ns" netns "$vnet" # bind mount needs to already exist
ip link set "$veth_host" up
ip link set "$veth_host" master "$br"
}
function link_vnet {
mkdir -p "$net_path"
if [ ! -f "$vnet_path" ]; then
# can also use bind mount, but it needs to happen after unshare
ln -s "${ns_path}/net" "$vnet_path"
chmod 0 "$vnet_path"
fi
}
function make_paths {
echo "Creating namespace paths."
mkdir -p "$merge_path" "$ns_path"
mount --bind "$ns_path" "$ns_path"
mount --make-rprivate "$ns_path"
for nsd in "${namespace_dirs[@]}"; do
touch "${ns_path}/${nsd}"
done
}
function start_services {
echo "Starting services."
mkdir -p /var/run/mongodb
/sbin/sshd 2> /dev/null
}
function enter_ns {
exec nsenter --net="${ns_path}/net" \
unshare --kill-child -f --mount-proc $namespace_flags_excl_net "$0" ns $container_num &
timeout 2 bash -c "while [[ -z \"$(pgrep -f "$pid_search")\" ]]; do sleep 0.1; done"
if [ $? != 0 ]; then
echo "Starting namespaces failed. Quitting."
exit 1
fi
echo "Entering namespaces in new shell."
exec "$0" up $container_num
}
function create_ns {
make_paths
link_vnet
echo "Adding net namespace ${vnet} and virtual device ${veth_host}."
unshare --net="${ns_path}/net" true
add_host_net
add_port_forwarding
enter_ns
}
function add_proxy_ports {
echo "Adding proxy ports: ${ports_web[@]}"
lines=$(
for port in "${ports_web[@]}"; do
printf "%-8s%8s;\n" $(($port + $container_num * $offset)) $port
done)
echo "${lines[@]}" > "/etc/nginx/conf.d/port-map${container_num}.map";
lines=$(
for port in "${ports_web[@]}"; do
printf "listen%8s ssl;\n" $(($port + $container_num * $offset))
done)
echo "${lines[@]}" > "/etc/nginx/conf.d/port${container_num}.conf";
}
function remove_proxy_ports {
rm "/etc/nginx/conf.d/port-map${container_num}.map" \
"/etc/nginx/conf.d/port${container_num}.conf" 2> /dev/null;
}
####################################################################
if [ "$cmd" = "delete" ]; then
pkill -9 -f "$pid_search"
unmount_ns
remove_vnet
remove_proxy_ports
rm -rf "$container_path"
elif [ "$cmd" = "kill" ]; then
pkill -9 -f "$pid_search"
elif [ "$cmd" = "up" ]; then
if [ ! -d "$container_path" ] || none_mounted; then
if [ -e "$vnet_path" ]; then
echo "The ${vnet_path} net namespace already exists. Deleting."
remove_vnet
fi
add_net
add_proxy_ports
nginx -s reload
create_ns
else
nsenter -F $namespace_flags --wd=/home/"$user" "${switch_user[@]}" #2> /dev/null
if [ $? = 1 ]; then
echo "Unable to enter namespaces. Attempting to unmount."
unmount_ns_excl_net
if ! none_mounted_excl_net; then
echo "Unable to unmount."
exit 1
fi
echo "Entering namespaces."
enter_ns
fi
fi
elif [ "$cmd" = "ns" ]; then
# container paths
upper="${merge_path}/upper"
work="${merge_path}/work"
merge="${merge_path}/root"
mkdir -p "$upper" "$work" "$merge"
# network
ip="${subnet}.${octet}"
device="$(ip addr ls "$veth_ns")"
if [[ ! "$device" =~ "$ip" ]]; then
ip addr add "${ip}/24" dev "$veth_ns"
ip link set "$veth_ns" up
ip route add default via "${subnet}.0"
ip link set up dev lo
sysctl net.ipv4.ping_group_range="0 2147483647" > /dev/null
sysctl net.ipv4.ip_unprivileged_port_start=0 > /dev/null
cp "/etc/hosts" "${merge_path}/hosts"
echo "${ip} ${hostname}" >> "${merge_path}/hosts"
fi
# mounts
mount -o ro,bind "${merge_path}/hosts" "/etc/hosts"
mount -t overlay overlay -o lowerdir=/,upperdir="$upper",workdir="$work" "$merge"
for op in "${overlay_paths[@]}"; do
mount --bind "${merge}${op}" "$op"
done
start_services
sleep infinity
fi
# UNCONFIGURED FSTAB FOR BASE SYSTEM
{{dev_path}} /home/share drvfs defaults,ro 0 0
C: /mnt/c drvfs defaults,ro 0 0
D: /mnt/d drvfs defaults,ro 0 0
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
#pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#access_log /var/log/nginx/access.log main;
client_max_body_size 100M;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 4096;
include /etc/nginx/mime.types;
default_type application/octet-stream;
map $ssl_client_s_dn $client_cert_file {
~CN=([^,]+) $1;
default '';
}
map $args $replace_host_args {
~(.*)\d+\.{{subdomain}}\.{{tld}}(.*) $1{{hostname}}$2;
default '';
}
map $host $server_num {
~^(\d+)\.{{subdomain}}\.{{tld}}$ $1;
~^{{subdomain}}(\d+)\.{{tld}}$ $1;
default '';
}
map $server_port $proxy_port {
include /etc/nginx/conf.d/port-map*.map;
default 0;
}
ssl_certificate "/home/{{username}}/certs/{{hostname}}.crt";
ssl_certificate_key "/home/{{username}}/certs/{{hostname}}.key";
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
ssl_ciphers PROFILE=SYSTEM;
ssl_prefer_server_ciphers on;
ssl_verify_client optional_no_ca;
proxy_http_version 1.1;
proxy_set_header Accept-Encoding "";
proxy_set_header Host {{hostname}}:$proxy_port;
proxy_redirect $scheme://{{hostname}} $scheme://$server_num.{{hostname}};
proxy_cookie_domain {{hostname}} $server_num.{{hostname}};
sub_filter {{hostname}} $server_num.{{hostname}};
sub_filter_once off;
sub_filter_types *;
# https://stackoverflow.com/questions/55361751/nginx-rewrite-of-query-params-with-multiple-replacements
# https://serverfault.com/questions/1091091/the-location-directive-has-no-effect-when-it-contains-a-variable
server {
set $new_req $scheme://$server_num.{{hostname}}:$proxy_port;
if ($host ~ ^{{subdomain}}(\d+)\.{{tld}}) { return 301 $new_req$request_uri; }
if ($replace_host_args !~ ^$) { return 307 $new_req$uri?$replace_host_args; }
if ($client_cert_file ~ ^$) { rewrite ^/(.*) /__no_such_path__/$1; }
include /etc/nginx/conf.d/port*.conf;
server_name $server_num.{{hostname}};
location / {
proxy_ssl_certificate "/home/{{username}}/certs/${client_cert_file}.pem";
proxy_ssl_certificate_key "/home/{{username}}/certs/${client_cert_file}.pem";
proxy_pass $scheme://{{subnet}}.$server_num:$proxy_port;
}
location ~ /__no_such_path__/.* {
internal;
rewrite ^/__no_such_path__/(.*) /$1 break;
proxy_pass $scheme://{{subnet}}.$server_num:$proxy_port;
}
}
}
# Run from directory with rocky image (ignore any mount errors after restart)
# wsl --import dev C:\dev (Get-Item Rocky-9-Container*).Name --version 2
# wsl -d dev dnf install -y ansible-core
# wsl -d dev LANG=C.UTF-8 ansible-playbook playbook-wsl.yml -e "username=myuser tld=rocky9 dev_path=C:/Dev wsl_win_path=$($PWD -replace '\\', '/')"
- hosts: localhost
gather_facts: false
become: true
vars:
subdomain: 'dev'
subnet: '192.168.200'
hostname: '{{subdomain}}.{{tld}}'
wsl_linux_path: '{{lookup("pipe","wslpath " + wsl_win_path)}}'
tasks:
- name: Install packages
dnf:
name: [sudo,glibc-langpack-en,procps,iproute,iputils,iptables,e2fsprogs,openssh-server,nginx,chrony,man,man-pages,rsync,git]
state: latest
- name: 'Create user {{username}} with root privileges'
block:
- user:
name: '{{username}}'
password: '{{username | password_hash("sha512")}}'
- copy:
content: '{{username}} ALL=(ALL) NOPASSWD:ALL'
dest: '/etc/sudoers.d/{{username}}'
- name: Copy /etc templates
block:
- getent:
database: passwd
- template:
src: '{{wsl_linux_path}}/{{item}}.j2'
dest: '/etc/{{item}}'
loop: ['fstab','wsl.conf','nginx/nginx.conf']
- name: Set locale
lineinfile:
path: '/etc/locale.conf'
regexp: 'LANG=".*"'
line: 'LANG=en_US.UTF-8'
- name: Networking
block:
- name: Update hosts file
lineinfile:
path: '/etc/hosts'
regexp: '(192\.168\.|127\.0\.1\.1).*'
state: absent
# you may need these as well
- when: false
shell: |
rm /etc/resolv.conf
printf "nameserver %s\n" 8.8.8.8 8.8.4.4 > /etc/resolv.conf
chattr +i /etc/resolv.conf
setcap cap_net_raw+ep /bin/ping
# for rsync
ssh-keygen -A; /sbin/sshd 2> /dev/null
- name: Copy container script
copy:
src: '{{wsl_linux_path}}/dev-container.sh'
dest: '/home/{{username}}'
owner: '{{username}}'
group: '{{username}}'
mode: 0744
# These will be set by wsl.conf after reboot
- block:
- name: Make share path
file:
path: /home/share
owner: '{{username}}'
group: '{{username}}'
state: directory
- name: Mount fstab
command: mount -a
- name: 'Set hostname to {{hostname}}'
command: 'hostname {{hostname}}'
# hostname:
# name: '{{hostname}}'
param(
[byte]$octet=0,
[switch]$remove=$false)
$listen_ip="192.168.${octet}.1"
$connect_ip="192.168.1.0"
$ports=22,80,443
$offset=1000
# Add IP to network
$netcfg = netsh.exe interface ip show addresses
if (!$remove -and $octet -ne 0) {
if (!($netcfg -match "192.168.${octet}.1")) {
Write-Host "Adding IP 192.168.${octet}.1 to network"
$adapterName = (Get-NetAdapter -IncludeHidden | Where-Object {$_.Name -match '^vEthernet.*WSL'}).Name
netsh.exe interface ip add address "${adapterName}" "192.168.${octet}.1" 255.255.255.0
}
}
# Update ports
foreach ($port in $ports) {
# always remove ports first
netsh interface portprox delete v4tov4 listenport="$port" listenaddress="$listen_ip" 1>$null 2>$null
if (!$remove -and (0 -lt $octet) -and ($octet -lt 255)) {
netsh interface portproxy add v4tov4 listenport="$port" listenaddress="$listen_ip" connectport="$($port+$offset*$octet)" connectaddress="$connect_ip" 1>$null 2>$null
}
}
netsh interface portproxy show v4tov4
{
"copyOnSelect": false,
"defaultProfile": "{61c54bbd-c2c6-5271-96e7-009a87ff44bf}",
"initialCols": 180,
"initialRows": 40,
"windowingBehavior": "useAnyExisting",
"profiles":
{
"defaults":
{
"colorScheme": "goldfita",
"historySize": 20000,
"bellStyle": "taskbar",
"hidden": true,
"suppressApplicationTitle": true,
"startingDirectory": "~",
"icon": "ms-appx:///ProfileIcons/{9acb9455-ca41-5af7-950f-6bca1bc9722f}.png"
},
"list":
[
{
"name": "Dev 1",
"commandline": "C:\\Windows\\system32\\wsl.exe -d rocky -u goldfita ./dev-container.sh up 1",
"hidden": false,
"guid": "{2c4de342-38b7-51cf-b940-2309a097f518}"
},
{
"name": "Dev 2",
"commandline": "C:\\Windows\\system32\\wsl.exe -d rocky -u goldfita ./dev-container.sh up 2",
"hidden": false,
"guid": "{764ac376-a20c-5599-b489-c4117f1d45f3}"
},
{
"name": "Dev Admin",
"commandline": "C:\\Windows\\system32\\wsl.exe -d rocky -u goldfita",
"hidden": false,
"guid": "{e59f6e23-392d-4912-bc02-26c7f128adad}"
},
{
"name": "Windows PowerShell",
"commandline": "%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
"hidden": false,
"guid": "{61c54bbd-c2c6-5271-96e7-009a87ff44bf}",
"icon": "ms-appx:///ProfileIcons/{61c54bbd-c2c6-5271-96e7-009a87ff44bf}.png"
},
{
"name": "Windows PowerShell Admin",
"commandline": "%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
"hidden": false,
"elevate": true,
"guid": "{8ddae478-6679-40b0-8044-ac47844101ac}",
"icon": "ms-appx:///ProfileIcons/{61c54bbd-c2c6-5271-96e7-009a87ff44bf}.png"
}
]
},
"schemes":
[
{
"name": "goldfita",
"background": "#0C0C0C",
"black": "#0C0C0C",
"blue": "#0037DA",
"brightBlack": "#767676",
"brightBlue": "#3B78FF",
"brightCyan": "#61D6D6",
"brightGreen": "#16C60C",
"brightPurple": "#B4009E",
"brightRed": "#E74856",
"brightWhite": "#F2F2F2",
"brightYellow": "#F9F1A5",
"cursorColor": "#767676",
"cyan": "#3A96DD",
"foreground": "#F58700",
"green": "#13A10E",
"purple": "#881798",
"red": "#C50F1F",
"selectionBackground": "#61D6D6",
"white": "#CCCCCC",
"yellow": "#C19C00"
}
]
}

WSL Containers

A script for creating multiple development environments in a single WSL instance.

./dev-container.sh 1 up

Script variables

  • user - User to run the container.
  • ports_excl_web - Ports excluding web.
  • ports_web - Web ports.
  • overlay_paths - Paths that will not be shared by containers.

Support scripts

  • port_forward.ps1 - Add port forwarding in Windows.
  • create_wsl_network.ps1 - Create WSL ethernet adapter.
[automount]
enabled = true
root = /mnt
options = "metadata"
mountFsTab = true
[network]
generateResolvConf = false
generateHosts = false
hostname = {{hostname}}
[interop]
appendWindowsPath = true
[boot]
command = /usr/sbin/chronyd; /usr/sbin/nginx
@goldfita
Copy link
Author

win-terminal-containers

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment