爬取 openwrt barrier_breaker 软件包源的 python 脚本

openwrt.py

#!/usr/bin/env python  
#coding=utf-8
#  
# Openwrt Package Grabber  
#  
# Copyright (C) 2014 http://shuyz.com   
#     

import urllib2 
import re
import os

# the url of package list page, end with "/"
baseurl = 'http://archive.openwrt.org/barrier_breaker/14.07/ar71xx/generic/packages/packages/'

# which directory to save all the packages, end with "/"
savedir = './download/'
if not os.path.exists(savedir):
    os.makedirs(savedir)

print 'fetching package list from ' + baseurl
content = urllib2.urlopen(baseurl, timeout=15).read()

print 'packages list ok, analysing...'
pattern = r'<td class="n"><a href="(.*?)">'
items = re.findall(pattern, content)

cnt = 0
for item in items:
	if item == '../':
		continue
	else:
		cnt += 1
		print 'downloading item %d: '%(cnt) + item
		if os.path.isfile(savedir + item):
			print 'file exists, ignored.'
		else:
			rfile = urllib2.urlopen(baseurl + item) 
			with open(savedir + item, "wb") as code:
				code.write(rfile.read())

print 'done!'