Download list of free software licenses from GNU page

free-software-licenses.csv

licenses.py

#!/usr/bin/env python
# coding: utf-8

# pip install requests rows[html]

from io import BytesIO
from urlparse import urljoin

import requests
import rows


LICENSES_URL = 'https://www.gnu.org/licenses/license-list.html'


def _convert_row(row):

    name = row.name.strip() if row.name else row.text
    name = ' '.join(line.strip() for line in name.splitlines())

    url = urljoin(LICENSES_URL, row.url)
    if url == LICENSES_URL:
        url = None

    short_name = row.short_name
    if not short_name:
        short_name = ''.join(name.split())

    return {'name': name,
            'short_name': short_name,
            'url': url, }


def download_licenses():

    response = requests.get(LICENSES_URL)

    # 'green', 'purple' and 'orange' are licenses free software licenses
    # (compatible with GPL, compatible with FDL and incompatible)
    table = rows.import_from_xpath(
            BytesIO(response.content),
            encoding=response.encoding,
            rows_xpath='//dl[@class="green" or @class="purple" or @class="orange"]/dt',
            fields_xpath={
                'name': './a[last()]/text()',
                'short_name': './a[last()]/@id',
                'text': './text()',
                'url': './a[last()]/@href', })

    # table[3:] is to remove 'legend' rows, which are blank
    return rows.import_from_dicts(map(_convert_row, table[3:]))


if __name__ == '__main__':
    licenses = download_licenses()
    rows.export_to_csv(licenses, 'free-software-licenses.csv')