scrape urls from an html file

scrape_urls.py

#!/usr/bin/env python3

"""Scrape all urls from an html document

"""

import os
import sys
import argparse

# pip install beautifulsoup4
from bs4 import BeautifulSoup


def main(arguments):

    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('infile', help="Input file", type=argparse.FileType('r'))
    parser.add_argument('-o', '--outfile', help="Output file",
                        default=sys.stdout, type=argparse.FileType('w'))

    args = parser.parse_args(arguments)

    doc = args.infile.read()
    soup = BeautifulSoup(doc, 'html.parser')

    for link in soup.find_all('a'):
        print(link.get('href'))



if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))