CURL Options and Versions

curl-options.csv

curl-options.py

import requests
import urlparse
from bs4 import BeautifulSoup
import re

url_base = "http://curl.haxx.se/libcurl/c/"
url = urlparse.urljoin(url_base, "easy_setopt_options.html")
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')

print "CURL_OPT,VERSION,LIBCURL_VERSION_NUM,URL"
for td in soup.table.find_all('td'):
    if td.a:
        url = urlparse.urljoin(url_base, td.a.get('href'))
        nm = td.a.get_text()
        pg = BeautifulSoup(requests.get(url).text, 'html.parser')
        # print url
        for h2 in pg.find_all('h2'):
            if h2.get_text() == "AVAILABILITY":
                p = h2.find_next_sibling('p')
                m = re.search("(\d+)\.(\d+)\.(\d+)", p.get_text())
                if m and m.group(1) and m.group(2) and m.group(3):
                    v1 = int(m.group(1))
                    v2 = int(m.group(2))
                    v3 = int(m.group(3))
                    ver = "%d.%d.%d" % (v1, v2, v3)
                    dfn = "0x%02x%02x%02x" % (v1, v2, v3)
                    var = nm
                    print "%s,%s,%s,%s" % (nm, ver, dfn, url)