/etc/init.d/nft2ipset: An nftables set to ipset synchronizer for use with OpenWRT/mwan3

nft2ipset

#!/bin/sh /etc/rc.common
# Start before firewall and mwan3 which are at Prio 19
START=18
APP=nft2ipset
USE_PROCD=1
SCRIPTPATH="/tmp/nft2ipset"

write_script() {
cat > "$1" <<'EOT'
#!/bin/sh
#check if the script is already running
PID=$$
SCRIPT="$(basename $0)"
TMPDIR="/tmp"
MONITORPIDFILE="$TMPDIR/$SCRIPT-$$.nftmonitorpid"
MONITORFIFO="$TMPDIR/$SCRIPT-$$.nftmonitorfifo"
mkfifo "$MONITORFIFO"

cleanup () {
  # Cleanup nft monitor subprocess
  if [ -f "$MONITORPIDFILE" ]; then
    MONITORPID="$(cat "$MONITORPIDFILE")"
    if [ "$MONITORPID" -gt 1 ]; then
      kill "$MONITORPID"
    fi
  fi
  # Remove pid file and fifo
  rm "$MONITORFIFO" "$MONITORPIDFILE"
}
trap cleanup TERM INT EXIT

create_or_update_ipset() {
  # Determine ipset parameters
  local DEF="$1"
  local NAME="$(echo "$DEF" | cut -d' ' -f2)"
  local OPTS=""
  local FAMILY="inet"
  if echo "$DEF" | grep -q "ipv6_addr"; then
    FAMILY="inet6"
    OPTS="$OPTS family $FAMILY"
  fi 
  local TIMEOUT="$(echo "$DEF" | sed -r 's/.*timeout ([0-9]*)s.*/\1/; t; s/.*/0/')"
  if [ -n "$TIMEOUT" -a "$TIMEOUT" -gt 0 ]; then
    OPTS="$OPTS timeout $TIMEOUT"
  fi

  # Create or update ipset from nftables set
  if [ "$(ipset list -n "$NAME")" = "$NAME" ]; then
    CUR="$(ipset list -t "$NAME")"
    if ! ( echo "$CUR" | grep -q "family $FAMILY"); then
      ( ipset destroy "$NAME" 2>&1 | logger -t "$SCRIPT" ) || logger -t "$SCRIPT" "WARNING: Could not destroy ipset with family != $FAMILY"    
    elif ! ( echo "$CUR" | grep -q "timeout $TIMEOUT"); then
      # Swap current iteration of the ipset with a new iteration due to timeout mismatch
      ipset create "_$NAME" hash:ip $OPTS
      ipset swap "_$NAME" "$NAME"
      ipset destroy "_$NAME"
      logger -t "$SCRIPT" "Replaced ipset $NAME with new iteration with timeout $TIMEOUT"
    fi
  fi
  if [ "$(ipset list -n "$NAME")" != "$NAME" ]; then
    # Create a new ipset with options matching the nftables set
    ipset create "$NAME" hash:ip $OPTS
    # Restart mwan3 if this ipset is used by it, it is already running but the set name is not found in active rule output
    if [ $? = 0 ] && grep -q "option ipset '$NAME'" /etc/config/mwan3 2>/dev/null && ( service | grep mwan3 | grep running ) && ( ! (mwan3 rules | grep -q "match-set $NAME" ) ); then
      mwan3 restart
    fi
    logger -t "$SCRIPT" "Created new ipset $NAME with timeout $TIMEOUT"
  fi

  # Add already existing entries to the set
  echo "$DEF" | sed -re 's/.*elements = \{ ([^\}]+) \}.*/\1/g; t; s/.*//g' | tr ',' '\n' | sed -re 's/^[ ]+//g;s/expires/timeout/g;s/s$//g' | while read LINE; do
    if [ -n "$LINE" ]; then
      ipset -q add "$NAME" $LINE && logger -t "$SCRIPT" "Added $LINE to $NAME upon ipset creation/update" || true
    fi
  done
}

# Check if ipsets exist for all currently existing nftsets or create otherwise
nft -nT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set" | while read DEF; do
  create_or_update_ipset "$DEF"
done

# Monitor nftables rule changes
nft -nT monitor > "$MONITORFIFO" 2>&1 &
echo $! > "$MONITORPIDFILE"
while read LINE; do
  if echo "$LINE" | grep -q "add element inet fw4"; then
    # Check if ipset exists or create otherwise
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    if [ "$(ipset list -n $NAME)" != "$NAME" ]; then
      DEF="$(nft -tnT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set $NAME")"
      create_or_update_ipset "$DEF"
    fi
    # Add element to ipset
    IP="$(echo "$LINE" | cut -d' ' -f 7)"
    EXPIRES="$(echo "$LINE" | sed -re 's/.*expires ([0-9]+)s.*/\1/; t; s/.*/0/')"
    ADDOPTS=""
    if [ $EXPIRES -gt 0 ]; then
      ADDOPTS="timeout $EXPIRES"
    fi
    if ipset -q test "$NAME" "$IP"; then
      # Refresh the entry by deleting it first if already existing
      ipset -q del "$NAME" "$IP"
      ipset -q add "$NAME" "$IP" $ADDOPTS
    else
      ipset -q add "$NAME" "$IP" $ADDOPTS
      logger -t "$SCRIPT" "Added $IP to ipset $NAME $ADDOPTS"
    fi
  elif echo "$LINE" | grep -q "add set inet fw4"; then
    # Create or update ipset 
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    DEF="$(nft -nT list sets | tr '\n' ' ' | awk '{$1=$1;print}' | sed -r 's/(set|table)/\n\1/g' | grep "^set $NAME")"
    create_or_update_ipset "$DEF"
  elif echo "$LINE" | grep -q "delete set inet fw4"; then
    # Clear and try to delete removed ipset (This will fail if it is in use by any iptables rule)
    NAME="$(echo "$LINE" | cut -d' ' -f 5)"
    ipset clear "$NAME"
    ipset destroy "$NAME" 2>&1 | logger -t "$SCRIPT"
  fi
done < "$MONITORFIFO"
EOT
}

start_service() {
  write_script "$SCRIPTPATH"
  chmod +x "$SCRIPTPATH"
  procd_open_instance
  procd_set_param command "$SCRIPTPATH"
  procd_set_param respawn
  procd_close_instance
}
service_stopped() {
  rm "$SCRIPTPATH"
}
# vim: ts=2 sw=2 et

When I am using mwan3 with tailscale and above solution, then also it's exit node functionality not working.
It took nearly 2 days got to know that mwan3 iptables can be the problem.
Finally need to remove mwan3.

Also created a script to replicate failover functionality using another script which you can use trigger using crontab. Sharing below if it help someone

#!/bin/bash
set -e
CHECK_IP=${CHECK_IP:-8.8.8.8}
PRIMARY_IF=${PRIMARY_IF:-prmwan}
BACKUP_IF=${BACKUP_IF:-bkpwan}
ACTIVE_METRIC=5
FAILOVER_METRIC=10

HEALTH_CHECK_FAIL=2

# Check if number of ping failed then retun false with interface
is_up() {
  network_name=$1
  interface=$(get_interface $network_name)
  if ping -I "$interface" -c$HEALTH_CHECK_FAIL "$CHECK_IP" &>/dev/null
  then
    echo true
  else
    echo false
  fi
}

is_primary_up() {
  is_up $PRIMARY_IF
}

is_backup_up() {
  is_ip $BACKUP_IF
}

# Get interface from network name
get_interface() {
  network_name=$1
  interface=""

  if [ $(uci get network.$network_name.proto) == "pppoe" ]; then
    interface=pppoe-"$network_name"
  else
    interface=$(uci get network.$network_name.device)
  fi
  echo $interface
}

activate_primary() {
  primary_metric=$(uci get network.$PRIMARY_IF.metric)
  backup_metric=$(uci get network.$BACKUP_IF.metric)
  if [ "$primary_metric" != "$ACTIVE_METRIC" ] || [ "$backup_metric" != "$FAILOVER_METRIC" ]; then
    uci set network.$PRIMARY_IF.metric=$ACTIVE_METRIC
    uci set network.$BACKUP_IF.metric=$FAILOVER_METRIC
    uci commit network
    /etc/init.d/network reload
    echo "Making $PRIMARY_IF active"
  fi
}

activate_backup() {
  primary_metric=$(uci get network.$PRIMARY_IF.metric)
  backup_metric=$(uci get network.$BACKUP_IF.metric)
  if [ "$backup_metric" != "$ACTIVE_METRIC" ] || [ "$primary_metric" != "$FAILOVER_METRIC" ] ; then
    uci set network.$BACKUP_IF.metric=$ACTIVE_METRIC
    uci set network.$PRIMARY_IF.metric=$FAILOVER_METRIC
    uci commit network
    /etc/init.d/network reload
    echo "Making $BACKUP_IF active"
  fi
}



if [ "$(is_primary_up)" = "true" ]; then
  activate_primary
else
  if [ "$(is_backup_up)" = "true" ]; then
    activate_backup
  fi
fi

BTW there are a couple of places where the code uses sed -r 's/(set|table)/\n\1/g' to parse nft's output but I think it doesn't handle the case where a nftable's name ends with 'set' or 'table'. I've tried replacing them with sed -r 's/ (set|table)/\n\1/g' which seems to work.