Skip to content

Instantly share code, notes, and snippets.

@shyal

shyal/test.py

Last active Oct 26, 2016
Embed
What would you like to do?
#!/usr/bin/env python
import requests
from argparse import ArgumentParser
import pickle
limit = 50
# getting urls and dumping them into file
def get_urls():
sites = requests.get("http://readthedocs.org/api/v1/project/?limit=%s&offset=0&format=json" % limit)
objects = sites.json()['objects']
links = ["http://readthedocs.org" + x['resource_uri'] for x in objects]
with open("links.p", "wb") as outfile:
pickle.dump(links, outfile)
def fetch_links():
with open("links.p", "rb") as infile:
links = pickle.load(infile)
import time
start = time.time()
for link in links:
response = requests.get(link)
print("url: %s, status code: %s" % (link, response.status_code))
print("url: %s, content: %s" % (link, response.text[0:50]))
print(time.time() - start)
# main function
def main():
parser = ArgumentParser(description="Perform proxy testing/URL list creation")
parser.add_argument("--urls", help="download and save urls ")
args = parser.parse_args()
# get urls
if args.urls:
get_urls()
else:
fetch_links()
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.