gistfile1.txt

def scrape_pb(url):
    url = URL_ROOT + url
    print(url)
    page = requests.get(url)
    tree = html.fromstring(page.text)
    if get_css(tree, "div.alert.alert-info"):
        continue
    else:
        medium = get_css(tree, "h1.resultheader-publicatietype")
        try:
            headline = get_css(tree, "h1.title")
        except:
            pass
        else:
            print(headline)
        times = {t.get("itemprop"): t.get("datetime") for t in tree.cssselect("time")}
        print(times)
        if "Datum vergadering" in times:
            date = times["Datum vergadering"]
            date = datetime.strptime(date, "%Y-%m-%d")
        else:
            date = times["datePublished"]
            date = datetime.strptime(date, "%d-%m-%Y %H:%M:%S")
        content, = tree.cssselect("div.agendapunt")
        body = content.text_content()
        return {"headline": headline,
                "text": body,
                "date": date,
                "medium": medium,
                "url": url}