Extract endpoints from a list of urls

endpointz.py

# ~/tools/endpointz.py urls.txt | tee endpoints.txt
import sys
from urllib.parse import urlparse

def extract_endpoints(urls):
	endpoints = set()
	for url in urls:
		parsed_url = urlparse(url)
		path = parsed_url.path.strip()
		if path:
			endpoints.add(path)
	return endpoints

def main():
	urls = sys.stdin.readlines()
	endpoints = extract_endpoints(urls)
	for endpoint in endpoints:
		sys.stdout.write(endpoint + '\n')

if __name__ == "__main__":
	main()