network-mods.php

#!/usr/bin/env php
<?php

$subnet = "167.235.212.72/29";
$gateway = "167.235.212.73";
$device = 'external';
$table = 'vswitch';

function echoit($command)
{
    echo $command . PHP_EOL;
    passthru($command);
}

function scopeit(string $route, string $ip): void {
    global $fullTable;
    if(!str_contains($fullTable, "default via $route")) {
        echoit("ip route add $ip dev internal scope link");
    }
}

$currentTable = trim(shell_exec("ip route show dev internal"));
$fullTable = trim(shell_exec("ip route show"));
if (!(str_contains($fullTable, "$subnet dev external"))) {
    // "bug" in the kernel -- reported, but mailing list said "not a bug"
    // we need to add an address in order to add rules, making this address pretty much unusable
    // this lets metallb work in L2 mode
    echoit("ip addr add $subnet dev $device");
    echoit("ip rule add from $subnet lookup $table");
    echoit("ip rule add to $subnet lookup $table");
    echoit("ip route add default via $gateway dev $device table $table");

    // add the route to an external node
    // workaround for https://github.com/cilium/cilium/issues/26588
    scopeit('65.108.66.65', '65.108.66.126'); // cameo
    scopeit('65.108.6.193', '65.108.6.254'); // cantor
    scopeit('65.108.75.193', '65.108.75.198'); // capital
}

// comment out if static routes don't work: initial workaround for https://github.com/cilium/cilium/issues/26588
return;

// get the list of pods
$podList = json_decode(shell_exec("kubectl get pods -n kube-system -o json"), true);

foreach ($podList['items'] as $pod) {
    // look for cilium pods
    if (str_starts_with($pod['metadata']['name'], 'cilium-')) {
        // skip the operator
        if (str_contains($pod['metadata']['name'], 'operator')) {
            continue;
        }
        printf("inspecting %s ...\n", $pod['metadata']['name']);

        // query cilium for the ip address
        $ip = shell_exec("kubectl exec -n kube-system {$pod['metadata']['name']} -- cilium status -o json");
        if(empty($ip)) {
            printf("no ip found for %s\n", $pod['metadata']['name']);
            continue;
        }
        $ip = json_decode($ip, true);

        // get the ip address block for the node
        $proxy = $ip['proxy']['ip'];
        $octets = explode('.', $proxy);
        $hostIp = match ($octets[2]) {
            '1', 1 => '10.0.1.0/24',
            '2', 2 => '10.0.2.0/24',
            '0', 0 => '10.0.0.0/24',
            default => throw new Exception("Unknown host octet: {$octets[2]}")
        };
        // if the route is already there, skip it
        if (str_contains($currentTable, $proxy)) {
            printf("proxy %s already configured\n", $proxy);
            continue;
        }

        // if this is the current host, cilium has it
        if(str_contains($fullTable, "$hostIp via $proxy dev cilium_host")) {
            printf("setting external routes for %s\n", $hostIp);
            continue;
        }

        // check if we need to delete the old route because it changed
        if (str_contains($currentTable, $hostIp)) {
            printf("host %s already configured\n", $hostIp);
            // we need to remove the route
            echoit("ip route del $hostIp dev internal");
        }

        // add the new route
        echoit("ip route add $hostIp via $proxy dev internal");
    }
}