diproxy.py

#!/usr/bin/env python
# DI's new licensing restriction setup is basically completely forgeable
# the only thing they actually enforce is referer checking
# 
# This program sits and does the necessary referer hacking, so that you can still listen to DI.fm on standard internet radio players
#
# TODO:
# - [x] get a full list of DI channels (maybe dynamically even, at boot?)
# - [ ] pretty-up the menu; jinja2?
# - [ ] print bandwidth reports as listeners join and leave
# - [x] the children seem to be themselves forking
# - [x] zombie child processes
#   - this is because we're never wait()ing on the children, so they never die;
#     but when is it the right time to wait()? really we want a step that is "wait() OR accept()"
#   - ah! there's a solution! when a child dies it sends SIGCHLD *to the parent*, which can then wait() to clean up.
#     this is somewhat version dependent though: Linux and BSD handle it differently. Maybe.
# - [x] The children should close() the listening socket before using the connected socket, so that they aren't holding it open longer than the parent
# - [x] add an option for the bind address
# - [ ] Something is wrong with the headers; Firefox plays fine, but
#       Android, Chrome, mplayer all buffer for up to 5 minutes before playing.

from __future__ import print_function

import os
import errno
from signal import *
import socket
import requests

from random import randint, choice
import uuid


# Scraped off of pub1.di.fm with this BeautifulSoup script:
# A 'better' idea is to do this scraping live at boot. But meh.
"""
#!/usr/bin/env python

from __future__ import print_function

import bs4
s = bs4.BeautifulSoup(open("di_channels.html"))
L = s(lambda e: e.find("h3") and e.find("h3").text.endswith("_aac"), class_="newscontent")
def extract_deets(div):
    code = div.find("h3",text=lambda e: e.startswith("Mount Point ")).text.replace("Mount Point /di_","").replace("_aac","")
    name = div.find("td",text="Stream Title:").nextSibling.text
    genre = div.find("td",text="Stream Genre:").nextSibling.text
    genre = genre.replace("Electronic ","").replace("Electronics ","")
    return tuple(str(e) for e in (code, name, genre)) # the coercion is because py2 is stupid about strings
L = [extract_deets(e) for e in L]
print("CHANNELS = {")
for code, name, genre in L:
    print("    %r: {'name': %r, 'genre': %r}," % (code, name, genre))
print("}")
"""

CHANNELS = {
    '00sclubhits': {'name': '00s Club Hits - DIGITALLY IMPORTED - The biggest dance floor anthems from the noughties!', 'genre': '00s Club Hits'},
    'ambient': {'name': 'Ambient - DIGITALLY IMPORTED - a blend of ambient, downtempo, and chillout', 'genre': 'Ambient Downtempo'},
    'atmosphericbreaks': {'name': 'Atmospheric Breaks - DIGITALLY IMPORTED - Spaced out, melodic and full of warmth - these broken beat dance tunes will keep you dazed and amused.', 'genre': 'Atmospheric Breaks'},
    'bassline': {'name': 'DI - Bassline', 'genre': 'Bassline Garage'},
    'bassnjackinhouse': {'name': "DI - Bass & Jackin' House", 'genre': "Bass & Jackin' House"},
    'bigbeat': {'name': 'DI - Big Beat', 'genre': 'Big Beat'},
    'bigroomhouse': {'name': 'DI - Big Room House', 'genre': 'House Big'},
    'breaks': {'name': 'Breaks - DIGITALLY IMPORTED - a fine assortment of trance and house breaks', 'genre': 'Breaks Trance'},
    'chillhop': {'name': 'DI - ChillHop', 'genre': 'ChillHop Downtempo Hip Hop'},
    'chillntropicalhouse': {'name': 'DI - Chill & Tropical House', 'genre': 'Chill Tropical House'},
    'chillout': {'name': 'Chillout - DIGITALLY IMPORTED - Full of trippy flavors, this channel is just what you need to relax.', 'genre': 'Chillout Ambient'},
    'chilloutdreams': {'name': 'Chillout Dreams - DIGITALLY IMPORTED - relax to the sounds of dream and ibiza style chillout', 'genre': 'Chillout Dream'},
    'chillstep': {'name': 'DI - Chillstep', 'genre': 'Chillstep Dubstep'},
    'classiceurodance': {'name': 'Classic EuroDance - DIGITALLY IMPORTED - Finest imported cheese on the net!', 'genre': 'Eurodance HiNRG Italo'},
    'classiceurodisco': {'name': 'DI - Classic EuroDisco', 'genre': 'Euro Italo Disco'},
    'classictrance': {'name': 'Classic Trance - DIGITALLY IMPORTED - relive the classic trance hits!!', 'genre': 'Trance Techno'},
    'classicvocaltrance': {'name': 'Classic Vocal Trance - DIGITALLY IMPORTED - Classic fusion of trance, dance, and chilling vocals together!', 'genre': 'Trance Pop'},
    'clubdubstep': {'name': 'DI - Club Dubstep', 'genre': 'Dubstep Club'},
    'clubsounds': {'name': 'Club Sounds - DIGITALLY IMPORTED - the hottest club and dance tunes 24/7', 'genre': 'Club Dance'},
    'darkdnb': {'name': 'DI - Dark DnB', 'genre': 'DnB Dark'},
    'darkpsytrance': {'name': 'DI - Dark PsyTrance', 'genre': 'PsyTrance Psy Trance Dark'},
    'deephouse': {'name': 'Deep House - DIGITALLY IMPORTED - Only the sexiest, silky smooth and groovy Deep House.', 'genre': 'House Deep'},
    'deepnudisco': {'name': 'Deep Nu-Disco - DIGITALLY IMPORTED - The new sounds of Deep Disco!', 'genre': 'House Disco'},
    'deeptech': {'name': 'DI - Deep Tech', 'genre': 'Tech Deep'},
    'detroithousentechno': {'name': 'DI - Detroit House & Techno', 'genre': 'Tech Deep'},
    'discohouse': {'name': 'Disco House  - DIGITALLY IMPORTED - Grooviest Disco House hits on the planet!', 'genre': 'Disco House'},
    'djmixes': {'name': 'DJ MIXES - DIGITALLY IMPORTED - non-stop DJ sets featuring various forms of techno & trance!', 'genre': 'Trance Techno'},
    'downtempolounge': {'name': 'DI - Downtempo Lounge', 'genre': 'Lounge Downtempo Jazz'},
    'drumandbass': {'name': 'Drum and Bass - DIGITALLY IMPORTED - tasty assortment to satisfy your drum and bass fix!', 'genre': 'Drum and Bass Jungle'},
    'drumstep': {'name': 'DI - Drumstep', 'genre': 'Drumstep Dubstep'},
    'dub': {'name': 'DI - Dub', 'genre': 'Dub Dubstep'},
    'dubstep': {'name': 'Dubstep - DIGITALLY IMPORTED - Dubstep hits and mixes!!!!', 'genre': 'Dubstep Garage'},
    'dubtechno': {'name': 'DI - Dub Techno', 'genre': 'Dub Techno'},
    'ebm': {'name': 'DI - EBM', 'genre': 'EBM'},
    'eclectronica': {'name': 'DI - EcLectronica', 'genre': 'Eclectronica Tech'},
    'electrohouse': {'name': 'Electro House - DIGITALLY IMPORTED - an eclectic mix of electro and dirty house', 'genre': 'Electro House'},
    'electronicpioneers': {'name': 'DI - Pioneers', 'genre': 'Classic New Age'},
    'electronics': {'name': 'DI - Electronics', 'genre': 'Electronics Techno House'},
    'electropop': {'name': 'DI - Electropop', 'genre': 'Electropop Electroclash'},
    'electroswing': {'name': 'DI - Electro Swing', 'genre': 'Electro Swing Retro'},
    'epictrance': {'name': 'DI - Epic Trance', 'genre': 'Trance Epic'},
    'eurodance': {'name': 'EuroDance - DIGITALLY IMPORTED - the newest and best of Eurodance hits', 'genre': 'Eurodance HiNRG Italo'},
    'funkyhouse': {'name': 'Funky House - DIGITALLY IMPORTED - A fine selection of funky house music!!', 'genre': 'House Disco'},
    'futurebass': {'name': 'Future Bass - DIGITALLY IMPORTED - Hard basslines, booming beats and insatiable grooves. Inspired by Trap, Juke and Garage - molded together into a unique booming style.', 'genre': 'Future Bass'},
    'futuregarage': {'name': 'DI - Future Garage', 'genre': 'Future Garage'},
    'futuresynthpop': {'name': 'Future Synthpop - DIGITALLY IMPORTED - Finest selection of futurepop and synthpop!!', 'genre': 'Industrial Synthpop'},
    'gabber': {'name': 'DI - Gabber', 'genre': 'Gabber Hardcore'},
    'glitchhop': {'name': 'DI - Glitch Hop', 'genre': 'Glitch Hop'},
    'goapsy': {'name': 'Goa & Psychedelic Trance - DIGITALLY IMPORTED - a voyage out of this world!', 'genre': 'Psychedelic Goa Trance'},
    'handsup': {'name': 'DI - Hands Up', 'genre': 'Hands Up Eurodance'},
    'hardcore': {'name': 'Hardcore - DIGITALLY IMPORTED - DJ mixes, hard dance and NuNRG!', 'genre': 'Hardcore Trance'},
    'harddance': {'name': 'Hard Dance - DIGITALLY IMPORTED - are you ready for this!', 'genre': 'Trance Techno'},
    'hardstyle': {'name': 'Hardstyle - DIGITALLY IMPORTED - Banging Hardstyle for your ears!!!', 'genre': 'Hardstyle Hard Bass'},
    'hardtechno': {'name': 'DI - Hard Techno', 'genre': 'Techno Hard'},
    'house': {'name': 'House - DIGITALLY IMPORTED - silky sexy deep house music direct from New York city!', 'genre': 'House Deep'},
    'idm': {'name': 'DI - IDM', 'genre': 'IDM Intelligent Dance'},
    'indiebeats': {'name': 'Indie Beats - DIGITALLY IMPORTED - Smooth, groovy and full of cutting edge, fresh ideas - beats to kick back and enjoy far from the club setting.', 'genre': 'Indie Beats'},
    'indiedance': {'name': 'DI - Indie Dance', 'genre': 'Indie Dance'},
    'jazzhouse': {'name': 'DI - Jazz House', 'genre': 'Jazz House'},
    'jungle': {'name': 'DI - Jungle', 'genre': 'Jungle D&B'},
    'latinhouse': {'name': 'Latin House - DIGITALLY IMPORTED - Finest selection of Latin house!!', 'genre': 'Latin House'},
    'liquiddnb': {'name': 'Liquid DnB  - DIGITALLY IMPORTED - Flowing with the freshest Liquid DnB!!', 'genre': 'Liquid Drum N Bass'},
    'liquiddubstep': {'name': 'DI - Liquid Dubstep', 'genre': 'Dubstep Liquid'},
    'liquidtrap': {'name': 'DI - Liquid Trap', 'genre': 'Liquid Trap'},
    'lounge': {'name': 'Lounge - DIGITALLY IMPORTED - sit back and enjoy the lounge grooves!', 'genre': 'Lounge Downtempo'},
    'mainstage': {'name': 'DI - Mainstage', 'genre': 'Mainstage Big House'},
    'melodicprogressive': {'name': 'DI - Melodic Progressive', 'genre': 'Melodic Progressive'},
    'minimal': {'name': 'Minimal - DIGITALLY IMPORTED - Finest selection of Minimal Techno & House!!', 'genre': 'Minimal Techno'},
    'nightcore': {'name': 'DI - Nightcore', 'genre': 'Nightcore'},
    'nudisco': {'name': 'DI - Nu Disco House', 'genre': 'Nu Disco House'},
    'oldschoolacid': {'name': 'Oldschool Acid - DIGITALLY IMPORTED - Oldschool sounds of Acid Techno, House, and Trance!', 'genre': 'Acid Techno'},
    'oldschoolelectronica': {'name': 'Oldschool Techno & Trance - DIGITALLY IMPORTED - old school techno, trance & rave!', 'genre': 'Techno Trance'},
    'oldschoolhouse': {'name': 'DI - Oldschool House', 'genre': 'House Oldschool'},
    'oldschoolrave': {'name': 'DI - Oldschool Rave', 'genre': 'Oldschool Rave'},
    'progressive': {'name': 'Progressive - DIGITALLY IMPORTED - house, techno, and trance beats for your mind!', 'genre': 'Progressive House Trance'},
    'progressivepsy': {'name': 'Progressive Psy - DIGITALLY IMPORTED - progressive psychedelic grooves', 'genre': 'Psychedelic Progressive'},
    'psybient': {'name': 'Psybient - DIGITALLY IMPORTED - The Psychedelic side of Ambient', 'genre': 'Psychedelic Ambient'},
    'psychill': {'name': 'PsyChill - DIGITALLY IMPORTED - downtempo psychedelic dub grooves, goa ambient, and world beats.', 'genre': 'Chillout Psy'},
    'russianclubhits': {'name': 'DI - Russian Club Hits', 'genre': 'Russian Club'},
    'soulfulhouse': {'name': 'Soulful House - DIGITALLY IMPORTED - house music selected from Paris with love!', 'genre': 'Soulful House Deep'},
    'spacemusic': {'name': 'Space Dreams - DIGITALLY IMPORTED - ambient space music for expanding minds', 'genre': 'Space Music Ambient'},
    'techhouse': {'name': 'Tech House - DIGITALLY IMPORTED - A fusion of techno and house with a deep, soulful vibe.', 'genre': 'Tech House'},
    'techno': {'name': 'Techno - DIGITALLY IMPORTED - From Minimal to Detroit to Schranz & all in between!', 'genre': 'Techno EDM'},
    'trance': {'name': "Trance Channel - DIGITALLY IMPORTED - we can't define it!", 'genre': 'Trance Techno'},
    'trap': {'name': 'DI - Trap', 'genre': 'Trap Crunk'},
    'tribalhouse': {'name': 'Tribal House - DIGITALLY IMPORTED - Finest selection of tribal and tech house!!', 'genre': 'Tribal House'},
    'umfradio': {'name': 'UMF Radio - DIGITALLY IMPORTED - UMF Radio', 'genre': 'UMF Ultra'},
    'undergroundtechno': {'name': 'DI - Underground Techno', 'genre': 'Techno Underground'},
    'vocalchillout': {'name': 'DI - Chillout Vocals', 'genre': 'Chillout Vocal'},
    'vocallounge': {'name': 'DI - Vocal Lounge', 'genre': 'Lounge Vocal'},
    'vocaltrance': {'name': 'Vocal Trance - DIGITALLY IMPORTED - a fusion of trance, dance, and chilling vocals together!', 'genre': 'Trance Pop'},
}


def menu(socket, address=None):
    socket.send(b"HTTP/1.1 200 OK\r\n\r\n")
    socket.send(b"<html><body>\r\n")
    socket.send(b"<h1>Menu</h1>\r\n")
    socket.send(b"<ul>\r\n")
    for channel in sorted(CHANNELS):
        socket.send(("<li><a href='%(channel)s'>%(genre)s</a>"%{'channel':channel, 'genre': CHANNELS[channel]['genre']}).encode("utf-8"))
    socket.send(b"</ul>\r\n")
    socket.send(b"</body></html>")


def stream(listener, channel, address=None, fmt="mp3"):
    """
    listener is the open tcp socket to an http client
    fmt is flv or mp3 (maybe ogg or m4a or aac too?)
    """
    #server = randint(1,7) # pub3.di.fm is down
    server = choice([1,2,4,5,6,7])
    listenerid = uuid.uuid4().hex
    url = "http://pub%d.di.fm/di_%s_aac?type=.%s&listenerid=%s&awparams=companionAds%%3Atrue" % (server, channel, fmt, listenerid)
    headers = {'Referer': 'http://www.di.fm/channels',
                'X-Requested-With': 'ShockwaveFlash/24.0.0.194',
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36', # TODO: randomize
               }

    print("GET %s" % url)
    for k,v in headers.items():
        print("%s: %s" % (k,v))
    print()
    req = requests.get(url, headers=headers, stream=True)

    # TODO: use contextlib.closing()
    if req.status_code == 200:
        # proxy the result
        listener.send(b"HTTP/1.1 200 OK\r\n")
        # headers
        for k,v in req.headers.items():
            listener.send(("%s: %s\r\n" % (k,v)).encode("utf-8"))
            print("%s: %s" % (k,v))
        print()
        listener.send(b"\r\n")
        listener.send(b"\r\n")
        # content
        # this runs until either the source (req) disconnects, the sink (listener) disconnects the network goes down, or forever...
        for byte in req.iter_content(settings.buffer_size):
            #print("sending %d to %s" % (len(byte), address)) # DEBUG
            try:
                listener.send(byte)
            except BrokenPipeError:
                # ...except that we terminate gracefully upon peer disconnection
                break
    else: 
        # wrap the error back to the user
        # headers
        listener.send(("HTTP/1.1 %d %s\r\n" % (req.status_code, req.reason)).encode("utf-8"))
        listener.send(b"Connection: close\r\n") # this should be unnecessary, but some http clients assume there will be AT LEAST one header, so this gives one, and a sensible one for the situation at that
        listener.send(b"\r\n")
        # body
        listener.send(b"<html><body>")
            # XXX XSS injection attack here:
        listener.send(("<h1>Error</h1><p>DI.fm stream <b><a href='http://di.fm/%(channel)s'>%(channel)s</a></b> could not be opened.</p>" % {'channel': channel}).encode("utf-8"))
        listener.send(b"<p>Would you like to <a href='/'>try</a> another?</p>")
        listener.send(b"</body></html>")

def child(listener, address):
    # read the headers off the browser
    headers = listener.recv(10000) # TODO: be smarter about reading initial headers
    print("Worker", os.getpid(), "connected to", address)
    print(headers.decode("utf-8")) # DEBUG
    headers = headers.decode("utf-8") #??
    headers = headers.split("\r\n") # XXX use StringIO so handle the ambiguity of \n vs \r\n line endings
    verb, path, _ = headers[0].split()
    assert verb == "GET"
    channel = path.split("/")[1]
    assert _ in ["HTTP/1.0", "HTTP/1.1"]

    try:
        if not channel:
            menu(listener, address)
        else:
            stream(listener, channel, address)
    finally:
        try:
            listener.shutdown(socket.SHUT_RDWR) # without this the socket never sends a FIN (or never sends it until the master socket closes??)
            listener.close() # but this is still necessary??
        except OSError as err:
            if err.errno == errno.ENOTCONN:
                # if the socket is already closed, then just leave it.
                # there doesn't seem to be a reliable way (in the C socket API, even!)
                # to figure out if a socket is closed or not beyond just trying to use it
                pass

                print(address,"disconnected")
            else:
                raise

#### Clean up exited children

# This design assumes 1:1 between a dead child and SIGCHLD
# it just calls wait() to get the most recently zombied child
def reap_zombie(_, frame):
    """
    Clean up zombie children when they are done.

    """
    os.wait()
#signal(SIGCHLD, reap_zombie)

# This design handles cleaning up *all* zombies because
# multiple SIGCHLDs can get compressed into a single one
# https://docs.oracle.com/cd/E19455-01/806-4750/signals-7/index.html
def reap_zombies(_, frame):
    """
    Clean up zombie children when they are done.

    """
    while True:
        try:
            pid, status = os.waitpid(-1, os.WNOHANG)
            if pid == 0:
                break
        except ChildProcessError as exc:
            # sometimes what should be pid==0 comes back as an error instead
            # I don't know why
            if exc.errno == 10:
                break
            else:
                raise
        print("%d exited with status %d" % (pid, status), flush=True)
signal(SIGCHLD, reap_zombies)

# this design makes the zombies get ignored by this program,
# which on most systems gets them reparented to init, which
# then handles reaping them
signal(SIGCHLD, SIG_IGN)


#### Main

if __name__ == '__main__':

    import argparse # why u so uglyamerican argparse¿?¿¿?
    settings = argparse.ArgumentParser(description="Proxy for DI.fm public streams at http://pub[1-7].di.fm, which have been locked off to only work with users using https://www.di.fm.")
    settings.add_argument("-H", "--host",  default="127.0.0.1", type=str, help="listen address. By default, only listens to localhost. Set to 0.0.0.0 to open directly to the public.") 
    settings.add_argument("-p", "--port", default=7171, type=int, help="listening port.") 
    settings.add_argument("-b", "--buffer_size", default=4*1024, type=int, help="how much data to buffer in bytes. A typial DI.fm stream is 64kb/s, or (8*1024)B/s; the default is 4*1024, which is about half a second.")
    settings = settings.parse_args()

    proxy = socket.socket()
    # SO_REUSEADDR so that this doesn't hold sockets open after a crash
    proxy.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    proxy.bind((settings.host, settings.port))
    proxy.listen()
    print("Go to http://%s:%d to listen" % (socket.gethostname() if settings.host=="0.0.0.0" else settings.host, settings.port))

    try:
        while True:
            # block the main process, waiting for listeners to connect
            listener, addr = proxy.accept()
            # when we get one, fork to a subprocess to do the proxying
            pid = os.fork()
            if pid == 0:
                # child process

                # shutdown the *listening* socket so that the child
                # cannot possibly accept more connections on it
                proxy.close()
                # request handler
                child(listener, addr)
                # exit cleanly, when the child is done (otherwise we loop back and by fork!)
                raise SystemExit(0)
            else:
                # parent process
                pass
    except KeyboardInterrupt:
        # graceful shutdown: not a real error
        pass
    finally:
        proxy.close()

To add HTTPs, my recommendation is to stick it behind a Real web server. Here's a nginx config snippet which forces HTTPs and proxies /di/ to diproxy/ (yes, that's two proxies involved; sorry, puritans).

http {
    server {
        listen 80;
        server_name radio;
        
        # diproxy, run as the radio user
        location /di/ {
            rewrite    /di/([^/]+) /$1 break;
            proxy_pass http://127.0.0.1:7171/; 
        }
        
        listen 443 ssl;
        ssl_certificate /path/to/fullchain.pem;
        ssl_certificate_key /path/to/privkey.pem;

        if ($scheme != "https") {
            return 301 https://$host$request_uri;
        }
    }

}

To deploy:

$ sudo adduser radio
$ sudo su radio
$ mkdir -p ~/.local/bin
$ wget https://gist.github.com/kousu/dc6484f38a5762452c6ac38d35a89824/raw/c1ec3f5e84077ccf81bc087c0e3b93bc68bf481f/diproxy.py ~/.local/bin/
$ chmod -R +x ~/.local/bin
$ echo 'PATH=~/.local/bin:$PATH' >> ~/.profile
$ crontab -e
@reboot     tmux new -d diproxy.py 
$ ^D
$ sudo reboot

There are a lot of good ideas about running daemons in http://dustin.sallings.org/2010/02/28/running-processes.html, but cron + tmux is just as good, and slightly better because you can interact with it by

$ sudo -U radio tmux attach

You could also use dtach or screen in place of tmux.

To get logging, add | tee diproxy.log.