Skip to content

Instantly share code, notes, and snippets.

@madjar
Created September 6, 2014 16:07
Show Gist options
  • Save madjar/6d0b614a991038cc59fa to your computer and use it in GitHub Desktop.
Save madjar/6d0b614a991038cc59fa to your computer and use it in GitHub Desktop.
import os
from collections import namedtuple
import subprocess
import tarfile
import re
import requests
from bs4 import BeautifulSoup
import jinja2
# First of all, we need to get a list of all packages in the kde release
SourcePackage = namedtuple('SourcePackage', 'name version url sha1')
def parse_package_line(soup):
"""Convert a line of the release page into a SourcePackage object"""
link, _, sha = soup.find_all('td')
name, version = link.text.rsplit('-', 1)
url = link.a['href']
sha1 = subprocess.check_output(['nix-hash', '--type', 'sha1',
'--to-base32', sha.text],
universal_newlines=True).strip()
return SourcePackage(name, version, url, sha1)
def fetch_package_list():
"""Returns a list of all packages in the kde release as SourcePackage
objects"""
page = requests.get('http://kde.org/info/kde-frameworks-5.0.0.php').text
soup = BeautifulSoup(page)
return[parse_package_line(line)
for line in soup.find_all('tr', valign="top")[1:]]
# Then, we need to download them to work on them locally
def package_path(package):
"""Downloads the package to the nix store if needed and returns the path"""
os.environ['PRINT_PATH'] = '1'
output = subprocess.check_output(['nix-prefetch-url', '--type', 'sha1',
package.url, package.sha1],
universal_newlines=True,
stderr=subprocess.DEVNULL)
_, path, _ = output.split('\n')
return path
# We need to look into the package tarball to get its metadata and
# dependencies
def parse_package_tarball(path):
with tarfile.open(path) as tar:
name = tar.getnames()[0]
readme_file = tar.extractfile('{}/README.md'.format(name))
description = extract_description(readme_file)
cmakelists_file = tar.extractfile('{}/CMakeLists.txt'.format(name))
dependencies = re.findall(r'find_package\((\w+)',
cmakelists_file.read().decode())
return description, dependencies
def extract_description(readme_file):
"""Parse a README.md file and return the description of the package"""
i = iter(readme_file)
for line in i:
if line == b'\n' or line.startswith(b'#'):
continue
# First non-empty, non-header line must be the one-line
# description. It is not exact, but good enough for now. Exact
# info could be extracted from
# http://api.kde.org/frameworks-api/frameworks5-apidocs/
return line.decode().strip()
class UnhandledDependency(Exception):
pass
def cmake_dep_to_nix(dep, all_packages_names):
"""Convert a dependency defined in CMakeLists.txt to a nix path"""
if dep == 'ECM':
return 'extra-cmake-modules'
if dep in ('XCB', 'X11_XCB'):
return 'libxcb'
if dep in ('X11', 'QCA2', 'ZLIB', 'FAM', 'BZzip2', 'Perl'):
return dep.lower()
if dep == 'PythonInterp':
return 'python2'
if dep == 'DocBookXML4':
return 'docbook_xml_dtd_45'
if dep == 'Phonon4Qt5':
return 'phonon'
if dep.startswith('Qt5'):
return 'qt5'
if dep.startswith('KF5'):
base = dep[3:].lower()
for prefix in ('', 'k', 'kde'):
name = prefix + base
if name in all_packages_names:
return name
raise UnhandledDependency(dep)
# We write the nix file for each package
def write_package_nix(package, all_packages_names):
path = package_path(package)
if package.name == 'extra-cmake-modules':
# The description cannot be extracted from the readme, and
# there are not dependencies
description = 'Extra CMake modules'
dependencies = []
else:
description, dependencies = parse_package_tarball(path)
buildInputs = [cmake_dep_to_nix(d, all_packages_names) for d in dependencies]
buildInputs = list(set(filter(bool, buildInputs)))
template = jinja2.Template("""
{ stdenv, fetchurl, cmake, {{args}}... }:
stdenv.mkDerivation rec {
name = "{{name}}-{{version}}";
src = fetchurl {
url = "{{url}}";
sha1 = "{{sha1}}";
};
buildInputs = [
cmake {{buildInputs}}
];
meta = {
# homepage = "http://minetest.net/";
description = "{{description}}";
# license = stdenv.lib.licenses.lgpl21Plus;
};
}
""")
result = template.render(args=', '.join(buildInputs + ['']),
name=package.name,
version=package.version,
url=package.url,
sha1=package.sha1,
description=description,
buildInputs=' '.join(buildInputs))
with open('{}.nix'.format(package.name), 'w') as f:
f.write(result)
# TODO : use real templates, and arguments to tell where to write
# Finally, we write default.nix
def write_default(packages):
lines = ['{ callPackage, xlibs }:', 'let pkgs = self: {']
lines.extend(' {0} = callPackage ./{0}.nix self // xlibs;'.format(p.name)
for p in packages)
lines.extend(['};', 'fixpkgs = pkgs fixpkgs;', 'in fixpkgs'])
with open('default.nix', 'w') as f:
f.writelines(l + '\n' for l in lines)
if __name__ == '__main__':
packages = fetch_package_list()
all_packages_names = {p.name for p in packages}
working_packages = []
for p in packages:
try:
write_package_nix(p, all_packages_names)
working_packages.append(p)
except UnhandledDependency as e:
print('Warning, not packaging {} because of unhandled dependency: {}'
.format(p.name, e.args[0]))
write_default(working_packages)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment