file download from web page

filedownload.py

# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import re
import urllib
import urllib2 as request
from bs4 import BeautifulSoup

def get_rpm_list(url):
    response = request.urlopen(url)
    body = response.read()
    soup = BeautifulSoup(body, "lxml")
    list = []
    pattern = re.compile(".*.rpm|.*.gz|.*.bz2|.*.xml")

    for link in soup.find_all('a'):
        text = link.get('href')
        #if text.find('.rpm') > -1:
        if pattern.match(text):
            list.append(text)
    return list

url = 'http://buildlogs.centos.org/centos/7/cloud/openstack-liberty/repodata/'
for item in get_rpm_list(url):
    urllib.urlretrieve(url + item, "E:/SoftWare/openstack/liberty/rpm/openstack-liberty/repodata/" + item)