Web-Scraping with Headless Chrome

headless_chrome_lxml.py

#!/usr/bin/env python3

import sys
import os
import subprocess
import lxml.html as html
import lxml.etree as etree

def get_page(url):
    HEADLESS_GOOGLE_CMD = '''google-chrome-stable --headless \
                                                  --disable-gpu \
                                                  --dump-dom'''
    target = '{} {}'.format(HEADLESS_GOOGLE_CMD, url)
    return subprocess.getoutput(target)


def parse_chunk(node):
    parts = [node.text]
    for c in node.getchildren():
        elem = None
        if c.tag == 'a' or c.tag == 'dfp-ad':
            elem = None
        elif c.tag == 'br':
            elem = '\n'
        else:
            assert False, c.tag
        parts.extend([c.text, elem, c.tail])

    if node.tag == 'br':
        parts.append('\n')

    parts.append(node.tail)
    parts = list(filter(None, parts))
    return ''.join(parts)



def parse_song(song_url):
    page_source = get_page(song_url)
    assert isinstance(page_source, str)
    root = html.fromstring(page_source)
    LYRICS_XPATH = "//div[@class='lyrics']/section/p"
    lyrics_html = root.xpath(LYRICS_XPATH)
    assert len(lyrics_html) == 1
    output = []
    for c in lyrics_html[0].getchildren():
        text = parse_chunk(c)
        output.append(text)
    return ' '.join(list(filter(lambda x : x, output)))


cd = 'https://genius.com/The-mamas-and-the-papas-california-dreamin-lyrics'
print(parse_song(cd))