ipk openwrt downloader

ipk.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

from requests import get
from lxml.html import document_fromstring
from os import path, makedirs
try:
    import progressbar  # pip install progressbar2
    bar = progressbar.ProgressBar()
    _bar = None
except Exception:
    bar = None

__author__ = 'Sergey Zharkov'
__license__ = 'MIT'
__email__ = 'sttv-pc@mail.ru'
__version__ = '0.0.1'
__url__ = 'https://github.com/1271'


uri = 'http://downloads.openwrt.org/chaos_calmer/15.05.1/ar71xx/generic/'
sub_dir = 'ar71xx/'


def _get_links_on_page(url):
    html = get(url).text
    items = document_fromstring(html).cssselect('#list tr td a')
    return [url + i.get('href') for i in items if i.get('href')[:2] != '..']


def _safe_downloader(url, dst):
    retries = 10
    while retries > 0:
        try:
            with open(dst, 'wb') as f:
                with get(url) as response:
                    f.write(response.content)
                    response.close()
                    f.close()
                    return
        except OSError:
            retries -= 1


def _save_file(url):
    filename = sub_dir + url[len(uri):]
    directory = path.dirname(filename)
    path.isdir(directory) or makedirs(directory)
    path.isfile(filename) or _safe_downloader(url, filename)


def _progress(current):
    if bar:
        _bar.update(current)


def _init_progress(count):
    if bar:
        global _bar
        _bar = bar(range(count))


def get_links(url):
    result = []
    items = _get_links_on_page(url)
    for i in items:
        if i[-1] == '/':
            result += get_links(i)
        else:
            result.append(i)
    _init_progress(len(result))
    return result


def save_files(urls):
    for i, url in enumerate(urls):
        _progress(i)
        _save_file(url)


if __name__ == '__main__':
    save_files(get_links(uri))