download.py

#!/usr/bin/python3
# Rex Tsai <rex.tsai@canonical.com>
# Requirement:
# 0. This script works only on Ubuntu classic.
# 1. apt-get install -y dctrl-tools python3-minimal git git-buildpackage
# 2. make sure you have the following apt source
# deb http://archive.ubuntu.com/ubuntu/ xenial main restricted multiverse universe
# deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted multiverse universe
# deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted multiverse universe
# deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted multiverse universe
# 3. apt-get update
#
# Known issue:
# Some packages are not imported as git repository, and it's been removed from apt archive.
# Please download the source from https://launchpad.net/ubuntu/+source/$PACKAGE/$VERSION
# And import with gbp import-dsc *.dsc

import sys
import yaml
import subprocess
import os

def sourcepkg(pkgname):
  (name,version)=pkgname.split('=')
  # Query the source package name of the package.
  pkg = subprocess.run("grep-dctrl -P %s --exact-match -s Source,Package /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial*_*_binary-amd64_Packages|head -1" % name, shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
  pkg = pkg.split(' ')
  name = pkg[1].strip()
  if(len(pkg)>2):
    version = pkg[2].strip().strip('(').strip(')')
  return (name, version)

def downloadPackage(name, version):
  if not os.path.exists(name):
    print("Downloading %s=%s" % (name, version))
    try:
      subprocess.run("git clone https://git.launchpad.net/~usd-import-team/ubuntu/+source/%s" % name, shell=True, check=True, env={'GIT_TERMINAL_PROMPT': '0'})
    except subprocess.CalledProcessError as err:
      # Unable to find an imported repository, download from apt archive.
      subprocess.run("apt-get source --download-only %s=%s && gbp import-dsc %s_%s.dsc && rm -rf %s_*orig*" % (name, version, name, version, name, version), shell=True, check=True)
  gitversion = version.replace(":","%").replace("~","_")
  subprocess.run("cd %s && git checkout -B %s $(git tag -l \*import/%s)" % (name, gitversion, gitversion), shell=True, check=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)

def downloadManifest(filename):
  manifest = yaml.load(open(filename, 'r'))
  for package in (manifest['parts']['hooks']['installed-packages']):
      (name, version) = sourcepkg(package)
      # Fixing version strings
      downloadPackage(name, version)
      
if __name__ == '__main__':
  if len(sys.argv) > 1:
    filename = sys.argv[1]
  else:
    filename = "/snap/core/current/snap/manifest.yaml"
  downloadManifest(filename)