Skip to content

Instantly share code, notes, and snippets.

@Zitrax
Last active July 18, 2023 18:30
Show Gist options
  • Save Zitrax/2bad212a3b0571357e1b to your computer and use it in GitHub Desktop.
Save Zitrax/2bad212a3b0571357e1b to your computer and use it in GitHub Desktop.
Download all dependencies for a ubuntu package.
import argparse
import hashlib
import os
from collections import defaultdict
import sys
from bs4 import BeautifulSoup as bs
import re
import requests
MD5 = r"^[a-fA-F\d]{32}$"
#
# Finds dependencies of a ubuntu package and downloads all
# needed packages. This might download more than needed
# since some packages can already be installed.
#
# However you can use 'dpkg -i -E *' to install them all
# while skipping already installed packages.
#
class UbuntuDeps(object):
def __init__(self, arch, release, mirror, fallback):
self.arch = arch
self.url = 'http://packages.ubuntu.com/' + release + "/"
self.mirror = mirror
self.fallback = fallback
@staticmethod
def download_file(_url, _dir):
local_filename = os.path.join(_dir, _url.split('/')[-1])
if not os.path.exists(local_filename):
if not os.path.exists(_dir):
os.mkdir(_dir)
r = requests.get(_url, stream=True)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return local_filename
@staticmethod
def md5(fname):
_hash = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
_hash.update(chunk)
return _hash.hexdigest()
@staticmethod
def verify_md5(expected, _dep):
if expected != deps[_dep]['md5']:
raise ValueError("MD5 check for " + _dep + " failed")
def find_dep(self, _dep, lvl=0, _deps=None):
if _deps is None:
_deps = defaultdict(lambda: defaultdict(dict))
if _dep in _deps:
return
print("*" * lvl + _dep)
r = requests.get(self.url + _dep)
soup = bs(r.text, 'html.parser')
for p in soup.select('p'):
if any(s in p.text for s in ["No such package", "Package not available in this suite"]):
print("\nERROR: Could not find package with name " + _dep + " Aborting!")
sys.exit(1)
if "This is a virtual package" in p.text:
prov = soup.select('div#pdeps dl dt a')
if prov:
print("*" * lvl + _dep + " -> " + prov[0].text)
_deps[_dep]['url'] = 'REDIR' # Mark as redirected
return self.find_dep(prov[0].text, lvl)
else:
print("\nERROR: Virtual package but could not parse providers. Aborting!")
# Find the dependencies
for dl in soup.select('ul.uldep dl'):
dt = dl.select('dt')[0]
a = dt.select('a')[0]
pkg = a.string
# noinspection PyStatementEffect
_deps[_dep]
self.find_dep(pkg, lvl + 1, _deps)
# Load download page
r = requests.get(self.url + self.arch + "/" + _dep + "/download")
soup = bs(r.text, 'html.parser')
for td in soup.select('table#pdownloadmeta td'):
if re.match(MD5, td.string):
_deps[_dep]['md5'] = td.string
# Find download link
found = False
for a in soup.findAll('a'):
if self.mirror in a['href']:
_deps[_dep]['url'] = a['href']
found = True
break
if not found:
for a in soup.findAll('a'):
if self.fallback in a['href']:
_deps[_dep]['url'] = a['href']
break
return _deps
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Download ubuntu dependencies',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter
(prog=sys.argv[0], max_help_position=80, width=120))
parser.add_argument('dep', help="The main/top package", nargs='+')
parser.add_argument('-a', '--arch', help="The architecture to use", default="amd64")
parser.add_argument('-r', '--release', help="Ubuntu release", default="trusty")
parser.add_argument('-m', '--mirror', help="Mirror to use for download", default="http://no.archive.ubuntu.com/")
parser.add_argument('-f', '--fallback', help="Mirror to use when main mirror is not found",
default="http://security.ubuntu.com/")
parser.add_argument('-d', '--directory', help="Target directory", default='pkg')
args = parser.parse_args()
ud = UbuntuDeps(arch=args.arch, release=args.release, mirror=args.mirror, fallback=args.fallback)
deps = None
for dep in args.dep:
print("\nDependency tree for " + dep + ":\n")
deps = ud.find_dep(dep, _deps=deps)
print("\nNeed to download:")
for dep in deps.keys():
print(dep + " : " + str(deps[dep]['url']))
print()
for dep in deps.keys():
if deps[dep]['url'] == 'REDIR':
continue
print("Downloading: " + dep)
ud.verify_md5(ud.md5(ud.download_file(deps[dep]['url'], args.directory)), dep)
@jimbju
Copy link

jimbju commented Jul 12, 2017

Hi, getting an error trying to download a package with a large number of deps, "mate-core".

ubuntu-deps.py -a amd64 -r xenial -d mate-core/ mate-core

Traceback (most recent call last):
File "ubuntu-deps.py", line 126, in
deps = ud.find_dep(dep, _deps=deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 85, in find_dep
self.find_dep(pkg, lvl + 1, _deps)
File "ubuntu-deps.py", line 91, in find_dep
if re.match(MD5, td.string):
File "/usr/lib/python2.7/re.py", line 141, in match
return _compile(pattern, flags).match(string)
TypeError: expected string or buffer

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment