This script acts as a lossy bash pipe to provide UDP style droppage when its more important to keep up with the latest messages than it is to actually process everything.

lossy.py

#!/usr/bin/env python3
# by: Cody Kochmann

import fcntl, os, sys

'''
This script acts as a lossy bash pipe to
provide UDP style droppage when its more
important to keep up with the latest messages
than it is to actually process everything.
'''

def setNonBlocking(fd):
    """
    Set the file description of the given file descriptor to non-blocking.
    (shamelessly ripped from twisted src)
    """
    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
    flags = flags | os.O_NONBLOCK
    fcntl.fcntl(fd, fcntl.F_SETFL, flags)

# set stdout to use nonblocking writes
setNonBlocking(1)
# optimize attribute lookups in python
write = sys.stdout.write
# silence stderr because python warns about 
# nonblocking writes failing. uncomment to see
sys.stderr = open('/dev/null')

# read stdin, in
for i in sys.stdin:
  try:
    write(i) # attempt to write to stdout
  except BlockingIOError:
    pass

'''
> # measure file size
> cat /var/log/messages | grep . -c
589999
> # thats how many total lines a lossless read should do
>
> # see how much slower grep is than cat
> cat /var/log/messages | ./lossy | grep . -c
554815
> # 6% loss
>
> # compare loss with reading from another grep
> # hopefully less loss because slower writer
> cat /var/log/messages | grep . | ./lossy | grep . -c
578066
> # 2% loss, because first grep feeds slower than cat
>
> # now what happens if we simulate a really slow reader?
> cat /var/log/messages | ./lossy | pv --rate-limit 5000000 --buffer-size 0 2>/dev/null | grep . -c
30354
> # 95% loss
'''