Get python code snippets from ddg directly from your terminal

ddg.py

#!/usr/bin/env python3

import ast
import sys
import requests
from bs4 import BeautifulSoup
import dbm
import unidecode
import subprocess

def cached_get(url, params={}):
    with dbm.open('get_cache', 'c') as db:
        key = f'{url}?{params}'
        if key in db:
            return db[key].decode('utf8')
        headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
        res =requests.get(url, params=params, headers=headers)
        db[key] = res.text.encode('utf8')
        return res.text

def cached_query(query):
    res = cached_get('https://duckduckgo.com/html/', params=dict(q=query))
    return res

def gather_code_candidates(root):
    for pre in root.find_all('pre'):
        parts = pre.contents
        text = ''.join('\n' if getattr(part, 'name', None) == 'br' else getattr(part, 'text', part) for part in parts)
        # to remove stupid quotes etc
        text = unidecode.unidecode(text)
        yield text

def gather_code(root):
    for text in gather_code_candidates(root):
        try:
            ast.parse(text)
            yield text
        except:
            pass

def get_snippets(query):
    html = cached_query(query)
    root = BeautifulSoup(html, 'html.parser')

    results = root.find_all('div', class_='result')

    for result in results:
        link = result.find('a')
        href = link.get('href')
        content = BeautifulSoup(cached_get(href), 'html.parser')
        for text in gather_code(content):
            text = text.strip()
            if not text:
                continue
            yield text

term_eraseline = subprocess.getoutput('tput el').rstrip('\n')
term_moveup = subprocess.getoutput('tput cuu 1').rstrip('\n')

if len(sys.argv) > 1:
    query = ' '.join(sys.argv[1:])
else:
    #print('Usage:', sys.argv[0], '<query>')
    query = subprocess.getoutput('xsel -b').splitlines()[0].strip()

print('QUERY:', repr(query))

try:
    try:
        for result in get_snippets(query):
            print(result)
            n = result.count('\n') + 2
            input()
            for i in range(n): sys.stdout.write(f'\r{term_eraseline}{term_moveup}{term_eraseline}')
    except EOFError:
        pass
    except KeyboardInterrupt:
        pass
except RuntimeError as ex:
    assert repr(ex) == "RuntimeError: generator ignored GeneratorExit"

$ ddg flask
QUERY: 'flask'
# save this as app.py
from flask import Flask

app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello, World!"

$  ddg connect two subprocesses via pipes python
QUERY: 'connect two subprocesses via pipes python'
grep_proc = subprocess.Popen(["grep", "rabbitmq"],
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE)
subprocess.Popen(["ps", "aux"], stdout=grep_proc.stdin)
out, err = grep_proc.communicate()