quick and dirty script for streaming ip2asn tsv and then saving the data locally as json

dump_ip2asn.py

#!/usr/bin/env python3
import netaddr
import json
from smart_open import open

def main():
    # Use smart_open to stream the compressed data line by line
    with open('./ip2asn.json', 'a+') as outfile:
        for line in open('https://iptoasn.com/data/ip2asn-v4.tsv.gz', encoding='utf-8'):
            range_start,range_end,AS_number,country_code,AS_description = line.split('\t')
            record = {
                'range_start': range_start,
                'range_start_int': int(netaddr.IPAddress(range_start)),
                'range_end': range_end,
                'range_end_int': int(netaddr.IPAddress(range_end)),
                'AS_number': AS_number,
                'country_code': country_code,
                'AS_description': AS_description.strip(),
                'cidr_ranges': []
            }
            # Populate cidr_ranges
            [record['cidr_ranges'].append(str(cidr)) for cidr in netaddr.iprange_to_cidrs(range_start, range_end)]
            outfile.write(f"{json.dumps(record)}\n")

if __name__ == '__main__':
    main()