recursively get web page links

recursively_get_links.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Time-stamp: < 1.py 2016-11-08 21:49:20 >
"""
递归抓取某个网站下的所有链接，结果输出到一个文件
"""

import requests
import time
import sys
import urllib
from bs4 import BeautifulSoup

header = {
	'User-Agent':'lixingcong_py_script',
	'Referer':'www.google.com',
	'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4'
}

all_text = ''
save_dir = '/tmp/'
save_filename_prefix = 'python_page'

def get_page(url):
	global all_text
	if url[-1] != '/':
		url += '/'

	human_readable_string = urllib.unquote(str(url))
	print human_readable_string

	conn = requests.get(url, headers=header, timeout=60)
	content = conn.content
	if content:
		res = BeautifulSoup(content, "html.parser")
		list1 = res.find_all('a')
		all_text += '---------------\n[DIR]: ' + human_readable_string + '\n'
		sub_dirs = []
		for i in list1:
			this_href = i['href']
			if this_href == '../':
				# print 'upper DIR'
				pass
			elif this_href[-1] == '/':
				sub_dirs.append(url + this_href)
			else:
				all_text += url + this_href + '\n'

		for sub_dir in sub_dirs:
			get_page(sub_dir)

	else:
		print "empty page"


if __name__ == '__main__':
	if len(sys.argv) != 2:
		print '\tusage: python ' + sys.argv[0] + ' URL'
		print '\n\tRecursively get all links for a url'
		exit(1)

	get_page(sys.argv[1])

	now = time.strftime("%H:%M:%S")
	filename = save_dir + save_filename_prefix +'_' + now + '.txt'

	with open(filename, 'w')as f:
		f.write(all_text)
		print '-----------\nresult was saved to ' + filename