Custom containers for development using Linux namespaces

create_wsl_network.ps1

# Deal with 'Logon failure: the user has not been granted the requested logon type at this computer.'
# This seems to BSOD Windows
#Get-Service vmcompute | Restart-Service

# Add the vEthernet adapter
wsl hostname -I

# Get IPs
$ips = New-Object System.Collections.Generic.HashSet[string]
$portproxy = netsh interface portproxy show v4tov4
foreach ($line in $portproxy)
{
    if ($line -match '^(\d.*?)\s.*$') {
        $ip = $Matches.1
        ($ips.Add($ip)) | out-null
    }
}
Write-Host $ips

# Add static IP to vEthernet adapter
foreach ($ip in $ips) {
    Write-Host $ip
    netsh.exe interface ip add address "vEthernet (WSL)" "${ip}" 255.255.255.0
}

dev-container.sh

#!/bin/bash

if [ ! -z "$VM_NUM" ]; then
   echo "This is already a container. Quitting."
   exit 1
fi
if [ "root" != "$USER" ]; then
    VM_NUM=$container_num source .bashrc
    exec sudo -E PATH="$PATH" "$0" "$@"
fi

cmd="$1"
container_num="$2"

containers_path="/containers"
container_path="${containers_path}/${container_num}"
merge_path="${container_path}/merge"
ns_path="${container_path}/ns"

vnet="vnet${container_num}"
veth_ns="veth${container_num}"
veth_host="br-veth${container_num}"
net_path="/var/run/netns"
vnet_path="${net_path}/${vnet}"
chain_forward="container${container_num}-forward"
chain_preroute="container${container_num}-preroute"

user=goldfita
hostname="$(hostname)"
br="br1"
eth="eth0"
offset=1000

subnet=192.168.200
namespace_dirs_excl_net=(mount pid)
namespace_dirs=("${namespace_dirs_excl_net[@]}" net)
ports_excl_web=(22)
ports_web=(80 443)
ports=("${ports_excl_web[@]}" "${ports_web[@]}")
overlay_paths=(/home/"$user" /var/log)

octet=$container_num
for nsd in "${namespace_dirs_excl_net[@]}"; do
    namespace_flags_excl_net="${namespace_flags_excl_net} --${nsd}=${ns_path}/${nsd}";
done
namespace_flags="${namespace_flags_excl_net} --net=${ns_path}/net"

pid_search="unshare.*${container_path}"
switch_user=(sudo -i --preserve-env="VM_NUM,WSL_DISTRO_NAME,WSL_INTEROP,PATH" -u "$user" VM_NUM=$container_num)

if [[ ! "$container_num" =~ [[:digit:]]+ ]]; then
    echo "You forgot to specify a container number (${container_num})."
    exit 1
fi

# Network namespaces
# https://superuser.com/questions/1715273/wsl2-two-separate-centos-distributions-have-same-eth0-inet-addres/1715457#1715457
# https://www.gilesthomas.com/2021/03/fun-with-network-namespaces

####################################################################

function none_mounted_excl_net {
    for nsd in "${namespace_dirs_excl_net[@]}"; do
        mountpoint -q "${ns_path}/${nsd}" && return 1
    done
    return 0
}

function none_mounted {
    mountpoint -q "${ns_path}/net" && return 1
    none_mounted_excl_net
    return $?
}

function unmount_ns_excl_net {
    for nsd in "${namespace_dirs_excl_net[@]}"; do
        umount "${ns_path}/${nsd}" 2> /dev/null
    done
}

function unmount_ns {
    unmount_ns_excl_net
    umount "$vnet_path" 2> /dev/null
    umount "${ns_path}/net" 2> /dev/null
    umount "$ns_path" 2> /dev/null
}

function remove_vnet {
    rm "$vnet_path" 2> /dev/null
    ip link delete "$veth_host" 2> /dev/null
}

function add_bridge {
    ip link add name br1 type bridge
    ip addr add "${subnet}.0/24" brd + dev "$br"
    ip link set "$br" up
}

function add_net {
    if [ ! -e /sys/devices/virtual/net/br1 ]; then
        ip addr add "192.168.1.0/16" dev "$eth" 2> /dev/null
        add_bridge
        add_forwarding
    fi
}

function add_forwarding {
    iptables -P FORWARD DROP
    iptables -A FORWARD -o eth0 -i "$br" -j ACCEPT
    iptables -A FORWARD -i eth0 -o "$br" -j ACCEPT
    iptables -t nat -A POSTROUTING -s "${subnet}.0/255.255.255.0" -o "$eth" -j MASQUERADE
    sysctl -w net.ipv4.ip_forward=1 > /dev/null
}

function add_port_forwarding {
    dest_ip="${subnet}.${octet}"
    # Add/replace chains
    iptables -F "$chain_forward" 2> /dev/null
    iptables -F "$chain_preroute" -t nat 2> /dev/null
    iptables -N "$chain_forward" 2> /dev/null
    iptables -N "$chain_preroute" -t nat 2> /dev/null
    if  [[ ! "$(iptables -S)" =~ $dest_ip ]]; then
        iptables -A FORWARD -d "${dest_ip}" -p tcp -m tcp -m state --state NEW,RELATED,ESTABLISHED -j "$chain_forward"
    fi
    if  [[ ! "$(iptables -S -t nat)" =~ ${eth}.*${chain_preroute} ]]; then
        iptables -t nat -A PREROUTING -i "$eth" -p tcp -m tcp -j "$chain_preroute"
    fi
    # These will be handled by nginx
    for port in "${ports_excl_web[@]}"; do
        new_port=$(($port + $container_num * $offset))
        iptables -t nat -A "$chain_preroute" -p tcp -m tcp --dport $new_port -j DNAT --to-destination "${dest_ip}:${port}"
        echo "Forwarding from ${subnet}.0:${new_port} to ${dest_ip}:${port}."
    done
    for port in "${ports[@]}"; do
        iptables -A "$chain_forward" -p tcp -m tcp --dport $port -j ACCEPT
    done
}

function add_host_net {
    ip link add "$veth_ns" type veth peer name "$veth_host"
    ip link set "$veth_ns" netns "$vnet" # bind mount needs to already exist
    ip link set "$veth_host" up
    ip link set "$veth_host" master "$br"
}

function link_vnet {
    mkdir -p "$net_path"
    if [ ! -f "$vnet_path" ]; then
        # can also use bind mount, but it needs to happen after unshare
        ln -s "${ns_path}/net" "$vnet_path"
        chmod 0 "$vnet_path"
    fi
}

function make_paths {
    echo "Creating namespace paths."
    mkdir -p "$merge_path" "$ns_path"
    mount --bind "$ns_path" "$ns_path"
    mount --make-rprivate "$ns_path"
    for nsd in "${namespace_dirs[@]}"; do
        touch "${ns_path}/${nsd}"
    done
}

function start_services {
    echo "Starting services."
    mkdir -p /var/run/mongodb
    /sbin/sshd 2> /dev/null
}

function enter_ns {
    exec nsenter --net="${ns_path}/net" \
         unshare --kill-child -f --mount-proc $namespace_flags_excl_net "$0" ns $container_num &
    timeout 2 bash -c "while [[ -z \"$(pgrep -f "$pid_search")\" ]]; do sleep 0.1; done"
    if [ $? != 0 ]; then
        echo "Starting namespaces failed. Quitting."
        exit 1
    fi
    echo "Entering namespaces in new shell."
    exec "$0" up $container_num
}

function create_ns {
    make_paths
    link_vnet
    echo "Adding net namespace ${vnet} and virtual device ${veth_host}."
    unshare --net="${ns_path}/net" true
    add_host_net
    add_port_forwarding
    enter_ns
}

function add_proxy_ports {
    echo "Adding proxy ports: ${ports_web[@]}"
    lines=$(
        for port in "${ports_web[@]}"; do
            printf "%-8s%8s;\n" $(($port + $container_num * $offset)) $port
        done)
    echo "${lines[@]}" > "/etc/nginx/conf.d/port-map${container_num}.map";
    lines=$(
        for port in "${ports_web[@]}"; do
            printf "listen%8s ssl;\n" $(($port + $container_num * $offset))
        done)
    echo "${lines[@]}" > "/etc/nginx/conf.d/port${container_num}.conf";
}

function remove_proxy_ports {
    rm "/etc/nginx/conf.d/port-map${container_num}.map" \
       "/etc/nginx/conf.d/port${container_num}.conf" 2> /dev/null;
}

####################################################################

if [ "$cmd" = "delete" ]; then

    pkill -9 -f "$pid_search"
    unmount_ns
    remove_vnet
    remove_proxy_ports
    rm -rf "$container_path"
   
elif [ "$cmd" = "up" ]; then
   
    if [ ! -d "$container_path" ] || none_mounted; then
        if [ -e "$vnet_path" ]; then
            echo "The ${vnet_path} net namespace already exists. Deleting."
            remove_vnet
        fi
        add_net
        add_proxy_ports
        nginx -s reload
        create_ns
    else
        nsenter -F $namespace_flags --wd=/home/"$user" "${switch_user[@]}" #2> /dev/null
        if [ $? = 1 ]; then
            echo "Unable to enter namespaces. Attempting to unmount."
            unmount_ns_excl_net
            if ! none_mounted_excl_net; then
                echo "Unable to unmount."
                exit 1
            fi
            echo "Entering namespaces."
            enter_ns
        fi
    fi
   
elif [ "$cmd" = "ns" ]; then

    # container paths
    upper="${merge_path}/upper"
    work="${merge_path}/work"
    merge="${merge_path}/root"
    mkdir -p "$upper" "$work" "$merge"
   
    # network
    ip="${subnet}.${octet}"
    device="$(ip addr ls "$veth_ns")"
    if [[ ! "$device" =~ "$ip" ]]; then
       ip addr add "${ip}/24" dev "$veth_ns"
       ip link set "$veth_ns" up
       ip route add default via "${subnet}.0"
       ip link set up dev lo
       sysctl net.ipv4.ping_group_range="0 2147483647" > /dev/null
       sysctl net.ipv4.ip_unprivileged_port_start=0 > /dev/null
       cp "/etc/hosts" "${merge_path}/hosts"
       echo "${ip} ${hostname}" >> "${merge_path}/hosts"
    fi
   
    # mounts
    mount -o ro,bind "${merge_path}/hosts" "/etc/hosts"
    mount -t overlay overlay -o lowerdir=/,upperdir="$upper",workdir="$work" "$merge"
    for op in "${overlay_paths[@]}"; do
        mount --bind "${merge}${op}" "$op"
    done
   
    start_services
    sleep infinity
fi

fstab.j2

# UNCONFIGURED FSTAB FOR BASE SYSTEM
{{dev_path}} /home/share drvfs defaults,ro 0 0
C:             /mnt/c      drvfs defaults,ro 0 0
D:             /mnt/d      drvfs defaults,ro 0 0

nginx.conf.j2

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
#pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    map $ssl_client_s_dn $client_cert_file {
        ~CN=([^,]+)  $1;
        default      '';
    }
    map $args $replace_host_args {
        ~(.*)\d+\.{{subdomain}}\.{{tld}}(.*)  $1{{hostname}}$2;
        default                   '';
    }
    map $host $server_num {
        ~^(\d+)\.{{subdomain}}\.{{tld}}$  $1;
        ~^{{subdomain}}(\d+)\.{{tld}}$    $1;
        default               '';
    }
    map $server_port $proxy_port {
        include /etc/nginx/conf.d/port-map*.map;
        default 0;
    }
   
    ssl_certificate           "/home/{{username}}/certs/{{hostname}}.crt";
    ssl_certificate_key       "/home/{{username}}/certs/{{hostname}}.key";
    ssl_session_cache         shared:SSL:1m;
    ssl_session_timeout       10m;
    ssl_ciphers               PROFILE=SYSTEM;
    ssl_prefer_server_ciphers on;
    ssl_verify_client         optional_no_ca;
   
    proxy_http_version        1.1;
    proxy_set_header          Accept-Encoding      "";
    proxy_set_header          Host                 {{hostname}}:$proxy_port;
    proxy_redirect            $scheme://{{hostname}}  $scheme://$server_num.{{hostname}};
    proxy_cookie_domain       {{hostname}}            $server_num.{{hostname}};
    sub_filter                {{hostname}}            $server_num.{{hostname}};
    sub_filter_once           off;
    sub_filter_types          *;

    # https://stackoverflow.com/questions/55361751/nginx-rewrite-of-query-params-with-multiple-replacements
    # https://serverfault.com/questions/1091091/the-location-directive-has-no-effect-when-it-contains-a-variable
    server {
       
        set $new_req $scheme://$server_num.{{hostname}}:$proxy_port;
        if ($host ~ ^{{subdomain}}(\d+)\.{{tld}}) { return 301 $new_req$request_uri; }
        if ($replace_host_args !~ ^$) { return 302 $new_req$uri?$replace_host_args; }
        if ($client_cert_file ~ ^$)   { rewrite ^/(.*) /__no_such_path__/$1; }

        include /etc/nginx/conf.d/port*.conf;
        server_name $server_num.{{hostname}};

        location / {
            proxy_ssl_certificate     "/home/{{username}}/certs/${client_cert_file}.pem";
            proxy_ssl_certificate_key "/home/{{username}}/certs/${client_cert_file}.pem";
            proxy_pass $scheme://{{subnet}}.$server_num:$proxy_port;
        }

        location ~ /__no_such_path__/.* {
            rewrite ^/__no_such_path__/(.*) /$1 break;
            proxy_pass $scheme://{{subnet}}.$server_num:$proxy_port;
        }

    }
}

playbook-wsl.yml

# Run from directory with rocky image (ignore any mount errors after restart)
# wsl --import dev C:\dev (Get-Item Rocky-9-Container*).Name --version 2
# wsl -d dev dnf install -y ansible-core
# wsl -d dev LANG=C.UTF-8 ansible-playbook playbook-wsl.yml -e "username=myuser tld=rocky9 dev_path=C:/Dev wsl_win_path=$($PWD -replace '\\', '/')"

- hosts: localhost
  gather_facts: false
  become: true

  vars:
    subdomain:         'dev'
    subnet:            '192.168.200'
    hostname:          '{{subdomain}}.{{tld}}'
    wsl_linux_path:    '{{lookup("pipe","wslpath " + wsl_win_path)}}'
   
  tasks:

    - name: Install packages
      dnf:
        name:  [sudo,glibc-langpack-en,procps,iproute,iputils,iptables,e2fsprogs,openssh-server,nginx,chrony,man,man-pages,rsync,git]
        state: latest
   
    - name: 'Create user {{username}} with root privileges'
      block:
        - user:
            name: '{{username}}'
            password: '{{username | password_hash("sha512")}}'
        - copy:
            content: '{{username}} ALL=(ALL) NOPASSWD:ALL'
            dest: '/etc/sudoers.d/{{username}}'
   
    - name: Copy /etc templates
      block:
        - getent:
            database: passwd
        - template:
            src: '{{wsl_linux_path}}/{{item}}.j2'
            dest: '/etc/{{item}}'
          loop: ['fstab','wsl.conf','nginx/nginx.conf']

    - name: Set locale
      lineinfile:
        path: '/etc/locale.conf'
        regexp: 'LANG=".*"'
        line: 'LANG=en_US.UTF-8'

    - name: Networking
      block:
        - name: Update hosts file
          lineinfile:
            path: '/etc/hosts'
            regexp: '(192\.168\.|127\.0\.1\.1).*'
            state: absent

        # you may need these as well
        - when: false
          shell: |
            rm /etc/resolv.conf
            printf "nameserver %s\n" 8.8.8.8 8.8.4.4 > /etc/resolv.conf
            chattr +i /etc/resolv.conf
            setcap cap_net_raw+ep /bin/ping
            # for rsync
            ssh-keygen -A; /sbin/sshd 2> /dev/null

    - name: Copy container script
      copy:
        src: '{{wsl_linux_path}}/dev-container.sh'
        dest: '/home/{{username}}'
        owner: '{{username}}'
        group: '{{username}}'
        mode: 0744

      
    # These will be set by wsl.conf after reboot
    - block:
        - name: Make share path
          file:
            path: /home/share
            owner: '{{username}}'
            group: '{{username}}'
            state: directory
           
        - name: Mount fstab
          command: mount -a
         
        - name: 'Set hostname to {{hostname}}'
          command: 'hostname {{hostname}}'
        #   hostname:
        #     name: '{{hostname}}'

port_forward.ps1

param(
    [byte]$octet=0,
    [switch]$remove=$false)

$listen_ip="192.168.${octet}.1"
$connect_ip="192.168.1.0"
$ports=22,80,443
$offset=1000

foreach ($port in $ports) {
    if ($remove) {
        netsh interface portproxy delete v4tov4 listenport="$port" listenaddress="$listen_ip"
    } elseif ((0 -lt $octet) -and ($octet -lt 255)) {
        netsh interface portproxy add v4tov4 listenport="$port" listenaddress="$listen_ip" connectport="$($port+$offset*$octet)" connectaddress="$connect_ip"
    }
}

netsh interface portproxy show v4tov4

wsl.conf.j2

[automount]
enabled = true
root = /mnt
options = "metadata"
mountFsTab = true

[network]
generateResolvConf = false
generateHosts = false
hostname = {{hostname}}

[interop]
appendWindowsPath = true

[boot]
command = /usr/sbin/chronyd; /usr/sbin/nginx