update_kf5.py

import os
from collections import namedtuple
import subprocess
import tarfile
import re
import requests
from bs4 import BeautifulSoup
import jinja2


# First of all, we need to get a list of all packages in the kde release

SourcePackage = namedtuple('SourcePackage', 'name version url sha1')

def parse_package_line(soup):
    """Convert a line of the release page into a SourcePackage object"""
    link, _, sha = soup.find_all('td')
    name, version = link.text.rsplit('-', 1)
    url = link.a['href']
    sha1 = subprocess.check_output(['nix-hash', '--type', 'sha1',
                                    '--to-base32', sha.text],
                                   universal_newlines=True).strip()
    return SourcePackage(name, version, url, sha1)


def fetch_package_list():
    """Returns a list of all packages in the kde release as SourcePackage
    objects"""
    page = requests.get('http://kde.org/info/kde-frameworks-5.0.0.php').text
    soup = BeautifulSoup(page)

    return[parse_package_line(line)
           for line in soup.find_all('tr', valign="top")[1:]]


# Then, we need to download them to work on them locally

def package_path(package):
    """Downloads the package to the nix store if needed and returns the path"""
    os.environ['PRINT_PATH'] = '1'
    output = subprocess.check_output(['nix-prefetch-url', '--type', 'sha1',
                                      package.url, package.sha1],
                                     universal_newlines=True,
                                     stderr=subprocess.DEVNULL)
    _, path, _ = output.split('\n')
    return path


# We need to look into the package tarball to get its metadata and
# dependencies

def parse_package_tarball(path):
    with tarfile.open(path) as tar:
        name = tar.getnames()[0]
        readme_file = tar.extractfile('{}/README.md'.format(name))
        description = extract_description(readme_file)

        cmakelists_file = tar.extractfile('{}/CMakeLists.txt'.format(name))
        dependencies = re.findall(r'find_package\((\w+)',
                                  cmakelists_file.read().decode())

    return description, dependencies

def extract_description(readme_file):
    """Parse a README.md file and return the description of the package"""
    i = iter(readme_file)
    for line in i:
        if line == b'\n' or line.startswith(b'#'):
            continue
        # First non-empty, non-header line must be the one-line
        # description. It is not exact, but good enough for now. Exact
        # info could be extracted from
        # http://api.kde.org/frameworks-api/frameworks5-apidocs/
        return line.decode().strip()


class UnhandledDependency(Exception):
    pass


def cmake_dep_to_nix(dep, all_packages_names):
    """Convert a dependency defined in CMakeLists.txt to a nix path"""

    if dep == 'ECM':
        return 'extra-cmake-modules'
    if dep in ('XCB', 'X11_XCB'):
        return 'libxcb'
    if dep in ('X11', 'QCA2', 'ZLIB', 'FAM', 'BZzip2', 'Perl'):
        return dep.lower()
    if dep == 'PythonInterp':
        return 'python2'
    if dep == 'DocBookXML4':
        return 'docbook_xml_dtd_45'
    if dep == 'Phonon4Qt5':
        return 'phonon'

    if dep.startswith('Qt5'):
        return 'qt5'


    if dep.startswith('KF5'):
        base = dep[3:].lower()
        for prefix in ('', 'k', 'kde'):
            name = prefix + base
            if name in all_packages_names:
                return name

    raise UnhandledDependency(dep)


# We write the nix file for each package

def write_package_nix(package, all_packages_names):
    path = package_path(package)

    if package.name == 'extra-cmake-modules':
        # The description cannot be extracted from the readme, and
        # there are not dependencies
        description = 'Extra CMake modules'
        dependencies = []
    else:
        description, dependencies = parse_package_tarball(path)

    buildInputs = [cmake_dep_to_nix(d, all_packages_names) for d in dependencies]
    buildInputs = list(set(filter(bool, buildInputs)))

    template = jinja2.Template("""
{ stdenv, fetchurl, cmake, {{args}}... }:

stdenv.mkDerivation rec {
  name = "{{name}}-{{version}}";

  src = fetchurl {
    url = "{{url}}";
    sha1 = "{{sha1}}";
  };

  buildInputs = [
    cmake {{buildInputs}}
  ];

  meta = {
#    homepage = "http://minetest.net/";
    description = "{{description}}";
#    license = stdenv.lib.licenses.lgpl21Plus;
  };
}
""")

    result = template.render(args=', '.join(buildInputs + ['']),
                             name=package.name,
                             version=package.version,
                             url=package.url,
                             sha1=package.sha1,
                             description=description,
                             buildInputs=' '.join(buildInputs))

    with open('{}.nix'.format(package.name), 'w') as f:
        f.write(result)

# TODO : use real templates, and arguments to tell where to write


# Finally, we write default.nix

def write_default(packages):
    lines = ['{ callPackage, xlibs }:', 'let pkgs = self: {']
    lines.extend('  {0} = callPackage ./{0}.nix self // xlibs;'.format(p.name)
                 for p in packages)
    lines.extend(['};', 'fixpkgs = pkgs fixpkgs;', 'in fixpkgs'])
    with open('default.nix', 'w') as f:
        f.writelines(l + '\n' for l in lines)


if __name__ == '__main__':
    packages = fetch_package_list()
    all_packages_names = {p.name for p in packages}
    working_packages = []
    for p in packages:
        try:
            write_package_nix(p, all_packages_names)
            working_packages.append(p)
        except UnhandledDependency as e:
            print('Warning, not packaging {} because of unhandled dependency: {}'
                  .format(p.name, e.args[0]))

    write_default(working_packages)