Create all network prefixes for a given CIDR to join against IPs

gistfile1.txt

from pyspark.sql.functions import udf, explode
from pyspark.sql.types import ArrayType, StringType

def get_networks(low, high):
    return [str(ipaddress.ip_address(x)) 
             for x in range(int(ipaddress.ip_address(low)), 
                int(ipaddress.ip_address(high)), 256)]

get_networks_udf = udf(get_networks, ArrayType(StringType()))

asns.select(explode(get_networks_udf(asns.ip_min, asns.ip_max))).show()

Sample:

+-----------+-----+--------+-----------+--------+
|       cidr|  asn|  ip_min|     ip_max|     col|
+-----------+-----+--------+-----------+--------+
| 1.0.0.0/24|15169| 1.0.0.0|  1.0.0.255| 1.0.0.0|
| 1.0.4.0/24|56203| 1.0.4.0|  1.0.4.255| 1.0.4.0|
| 1.0.5.0/24|56203| 1.0.5.0|  1.0.5.255| 1.0.5.0|
| 1.0.6.0/24|56203| 1.0.6.0|  1.0.6.255| 1.0.6.0|
| 1.0.7.0/24|56203| 1.0.7.0|  1.0.7.255| 1.0.7.0|
|1.0.20.0/23| 2519|1.0.20.0| 1.0.21.255|1.0.20.0|
|1.0.20.0/23| 2519|1.0.20.0| 1.0.21.255|1.0.21.0|
|1.0.22.0/23| 2519|1.0.22.0| 1.0.23.255|1.0.22.0|
|1.0.22.0/23| 2519|1.0.22.0| 1.0.23.255|1.0.23.0|
|1.0.24.0/23| 2519|1.0.24.0| 1.0.25.255|1.0.24.0|
|1.0.24.0/23| 2519|1.0.24.0| 1.0.25.255|1.0.25.0|
|1.0.26.0/23| 2519|1.0.26.0| 1.0.27.255|1.0.26.0|
|1.0.26.0/23| 2519|1.0.26.0| 1.0.27.255|1.0.27.0|
|1.0.28.0/22| 2519|1.0.28.0| 1.0.31.255|1.0.28.0|
|1.0.28.0/22| 2519|1.0.28.0| 1.0.31.255|1.0.29.0|
|1.0.28.0/22| 2519|1.0.28.0| 1.0.31.255|1.0.30.0|
|1.0.28.0/22| 2519|1.0.28.0| 1.0.31.255|1.0.31.0|
|1.0.38.0/24|24155|1.0.38.0| 1.0.38.255|1.0.38.0|
|1.0.39.0/24|24155|1.0.39.0| 1.0.39.255|1.0.39.0|
|1.0.64.0/18|18144|1.0.64.0|1.0.127.255|1.0.64.0|
+-----------+-----+--------+-----------+--------+