alienvault scraper python

alienvaultscraper.py

import argparse
import requests

# Define the API endpoint and parameters

base_url = "https://otx.alienvault.com/api/v1/indicators/domain/{domain}/url_list?limit=500&page={page}"
current_domain = ""

def make_request(domain, page):
    while True:
        current_domain = domain

        formatted_url = base_url.format(domain=current_domain, page=page)
        data = requests.get(formatted_url).json()

        has_next = data['has_next']

        for url_info in data["url_list"]:
            print(url_info["url"])

        if not has_next:
            break  # Exit the loop when has_next is False

        page = page + 1


if __name__ == "__main__":
    # Initialize argument parser
    parser = argparse.ArgumentParser(description="Fetch URLs associated with a domain from AlienVault OTX")

    # Add domain argument
    parser.add_argument("domain", type=str, help="The domain for which to fetch URLs (e.g., qoo10.jp)")

    # Parse command-line arguments
    args = parser.parse_args()

    make_request(domain=args.domain, page=1)