Scripts for acquisition of video on Raspberry Pi with transmssion to remote server via sshfs

capture.py

#!/usr/bin/env python3

"""
capture.py - A script to capture video and manage transfer of video data

Still in early stages of development and rapidly evolving to meet our needs. It
is partly a wrapper for raspivid, which it deploys.

Usage:
  capture.py [options]
  capture.py baseconfig
  capture.py -h | --help
  capture.py -v | --version

Options:
  -b --bitrate=<bps>         Bitrate in bits per second, recognizes common
                             SI prefixes [kib, kiB, Mib, MiB] and metric
                             prefixes [kb, kB, Mb, MB]. So 4kib is the same
                             as putting 4096. 0 is to leave unset [default: 0]
  -c --config=<conf-file>    Specify a config file to use. Will check for
                             ./capture.conf if not supplied
  -s --segment-ms=<ms>       The length of stream segments in milliseconds
  -w --segment-wrap=<n>      The segment number  max, wrap back to 1 after
  -r --raspivid-opts=<opts>  Takes a comma-separated list of arbitrary
                             options to pass into the raspivid command.
                             Options with arguments use the "=" to assign.
                             Example: vflip,rotation=15,exposure=verylong
  -h --help                  Show this screen.
  -v --version               Show version.
"""

from docopt import docopt
import asyncio
from configparser import ConfigParser, ExtendedInterpolation
from datetime import datetime
import functools
import json
from math import log10
import os
import re
import shutil
import subprocess
import sys
import time

__version__ = 'betadyne'

example_conf = """\
[capture.py]
--segment-ms = 2000       ; millisecond length of segments
--segment-wrap = 10000    ; highest segment number before wrap
--remote-fs = ./remotefs  ; path to remote filesystem mount
;The following option allows you to set arbitrary options on the call to
;raspivid. Use the longform name with no leading dashes, options with arguments
;are set with '='. Separate multiples with commas. Example is given. Note that
;capture.py does not check these for errors, so be careful!
;--raspivid-opts = vflip,rotation=15,exposure=verylong,save-pts
--raspivid-opts =
"""


def parse_bitrate(bitrate_str):
    unit_map = {'kiB': 8192,
                'MiB': 8388608,
                'kB' : 8000,
                'MB' : 8000000,
                'kib': 1024,
                'Mib': 1048576,
                'kb' : 1000,
                'Mb' : 1000000}
    myre=re.compile('(?P<val>\d+)(?P<unit>kb|kB|kib|kiB|Mb|MB|Mib|MiB)?')
    match = myre.match(bitrate_str)
    if match is None:
        raise ValueError('Invalid bitrate value: {}'.format(bitrate_str))
    val, unit = int(match.group('val')), match.group('unit')
    if unit is None:
        return val
    else:
        return unit_map[unit] * val

def merge_conf(conf1, conf2):
    """
    Merge two config dictionaries, with truthy values overriding falsy values.
    `conf1` takes priority over `conf2`.
    """
    return dict((str(key), conf1.get(key) or conf2.get(key))
        for key in set(conf2) | set(conf1))


def write_parameters(config):
    params_f = os.path.join(config['--remote-fs'], 'capture_parameters.json')
    print(params_f)
    with open(params_f, 'w') as outfile:
        json.dump(config, outfile)


async def check_process(process, loop):
    while process.poll() is None:  # Process still running if poll returns None
        if not loop.run:
            print('Killing the process!')
            process.terminate()
        await asyncio.sleep(2)
    print('Process has ended with returncode: '+str(process.returncode))
    loop.run = False


def file_closed(fp):
    res = subprocess.run(['lsof', fp], stdout=subprocess.PIPE)
    if res.stdout != '':
        return True
    else:
        return False


async def check_and_move_files(config, loop):
    last_time = time.time()
    while loop.run:
        now = time.time()
        last_time = now
        sleep = max(0, 2 - (now - last_time))
        await asyncio.sleep(sleep)
        #First we'll check for the poisonpill file
        if os.path.isfile('poisonpill'):
            os.remove('poisonpill')
            loop.run = False
        #Now we want to check if we are connectable
        findmnt = subprocess.run(['findmnt', '-M', config['--remote-fs']], stdout=subprocess.PIPE)
        if findmnt.stdout == '':  # This means it was not found!
            continue
        #Collect all current .h264 files and check that they are closed
        files = [f for f in sorted(os.listdir('.')) if os.path.splitext(f)[1]=='.h264']
        completed = [f for f in files if file_closed(f)]
        print(completed)
        for f in completed:
            remote_tmp = os.path.join(config['--remote-fs'], f)
            remote_complete = os.path.join(config['--remote-fs'], 'finished', f)
            try:
                cp = functools.partial(shutil.copy, f, remote_tmp)
                res = await loop.run_in_executor(None, cp)
            except:  # if there's an error, let's assume connection error
                continue
            os.rename(remote_tmp, remote_complete)
            os.remove(f)


def main():
    args = docopt(__doc__, version=__version__)

    if args['baseconfig']:
        with open('capture.conf', 'w') as outfile:
            outfile.write(example_conf)
        sys.exit(0)

    #Set up our configuration
    #We'll use the config file if given, capture.conf is default
    confp = ConfigParser(interpolation=ExtendedInterpolation(),
                         inline_comment_prefixes=';',
                         allow_no_value=True)
    if args['--config'] is None:
        print('Setting default config')
        args['--config'] = 'capture.conf'
    if os.path.isfile(args['--config']):
        with open(args['--config'], 'r') as conffile:
            confp.read_file(conffile)
        config = merge_conf(args, confp['capture.py'])
    else:
        print('No config file found! Using defaults')
        confp.read_string(example_conf)
        config = merge_conf(args, confp['capture.py'])

    #Type setting for variables, with some checking
    config['--segment-ms'] = int(config['--segment-ms'])
    assert config['--segment-ms'] > 0
    config['--segment-wrap'] = int(config['--segment-wrap'])
    assert config['--segment-wrap'] > 0
    config['--bitrate'] = parse_bitrate(config['--bitrate'])

    #Set the name format for the files, just sets the name by the current time
    now = datetime.now().isoformat()
    digits = int(log10(config['--segment-wrap'])) + 1
    name_format = '{}_%0{}d.h264'.format(now, digits)
    config['--name-format'] = name_format

    #Writes the parameters to a json file in the remotefs so that the
    #processing script can parse them for its functions
    write_parameters(config)

    #Compose the raspivid command
    raspivid_command = ['raspivid',
                        '--segment', str(config['--segment-ms']),
                        '--wrap', str(config['--segment-wrap']),
                        '--timeout', '0',
                        '--output', config['--name-format']]
    if config['--bitrate'] > 0:
        raspivid_command += ['--bitrate', str(config['--bitrate'])]
    if config['--raspivid-opts'] != '':
        for fullopt in config['--raspivid-opts'].split(','):
            if '=' in fullopt:
                opt, val = opt.split('=')
                raspivid_command += ['--'+opt, val]
            else:
                raspivid_command.append('--'+fullopt)
    print(raspivid_command)
    raspivid_proc = subprocess.Popen(raspivid_command,
                                     stdout=subprocess.DEVNULL,
                                     stderr=subprocess.DEVNULL)
    loop = asyncio.get_event_loop()
    loop.run = True

    try:
        loop.run_until_complete(asyncio.gather(check_process(raspivid_proc, loop),
                                               check_and_move_files(config, loop)))
    finally:
        subprocess.call(['touch', os.path.join(config['--remote-fs'], 'capture_done')])
        loop.close()
        raspivid_proc.terminate()


if __name__ == '__main__':
    main()

vidcapman.sh

#!/bin/bash

#Video Capture Manager
#Script to manage the capturing of video and transmission of data to remote
#server

#PIDFILE will be used to track background process ids
PIDFILE="pids.txt"
touch $PIDFILE

#echo $$ >> $PIDFILE

#SEGMENT_MS defines the segment length in milliseconds
SEGMENT_MS=2000  # 2 seconds
#WRAP defines the when the segment naming wraps back to 1
WRAP=10000
#NAME_FMT
NAME_FMT="video%05d"
#REMOTEFS defines the path to the remote filesystem
REMOTEFS="remotefs"
#FINISHED defines the path to a directory to store finished segments
FINISHED="finished"
#BITRATE defines the bitrate of video to produce in bits per second
#Filesize generally expressed with binare SI prefix (kibi, mebi, gibi...)
#Consider these values for file storage on remote server, but know that
#your final bitrate may vary if you transcode (not just stream copy)
kiB=8192    # 1 kiB =    1024 B =    8192 b = 1.0240 kB
MiB=8388608 # 1 MiB = 1048576 B = 8388608 b = 1.0486 MB
kB=8000     # 1 kB  =    1000 B =    8000 b = 0.9766 kiB
MB=8000000  # 1 MB  = 1000000 B = 8000000 b = 0.9537 MiB
#Network speeds generally expressed in Mbps (megabits per second, you don't
#want to exceed your file transfer speed limit!
kib=1024    # kibibit
Mib=1048576 # mebibit
kb=1000     # kilobit
Mb=1000000  # megabit
BITRATE=$(expr 8 \* $Mb)  # 8 megabits per second
#BITRATE=$(expr 500 \* kb)  # 500 kilobits per second

function on_exit {
    #Runs when script ends for any reason
    echo "Killing background processes"
    kill $(cat $PIDFILE)
}
trap on_exit EXIT

function check_finished_files {
    #Checking for .h264 files that have been closed and are thus finished
    if [ ! -d $FINISHED ]; then
        mkdir -p $FINISHED
    fi
    lsof_done=false
    for f in *.h264; do
        if ! $lsof_done; then
            lsof > open_files.txt
            lsof_done=true
        fi
        if ! [[ `grep $f open_files.txt` ]]; then
            echo "$f is closed, moving it to finished"
            mv $f $FINISHED/$f
        else
            echo "$f is open, skipping it"
        fi
    done
}

function transfer_finished_files {
    #Moves finished files into the remote filesystem and deletes them if successful
    if [[ $(findmnt -M $REMOTEFS) ]]; then
        :  # Proceed because we are connectable
    else
        return  # Abort because we are not connectable
    fi
    for f in ./finished/*.h264; do
        cp $f $REMOTEFS/$(basename $f)
        if [[ $? -eq 0 ]]; then
            echo "Successful transfer of $f"
            mv $REMOTEFS/$(basename $f) $REMOTEFS/$FINISHED/$(basename $f)
            rm $f
        else
            echo "Unsuccessful transfer of $f"
            return # Abort because we are not connectable
        fi
    done
}

#Start the video collection process, and track its process ID
echo "Starting video collection process..."
raspivid --vflip -sg "$SEGMENT_MS" -o $NAME_FMT".h264" -b $BITRATE -wr $WRAP -t 0 --save-pts &
echo $! > $PIDFILE
VIDPID=$!

#Wait a second and check that it has initialized successfully
sleep 1
if ps -p $VIDPID > /dev/null; then
    echo "Video collection is running"
else
    echo "Video has died, exiting"
    exit 1
fi

SECONDS=0
while true; do
    SLEEP=$(expr 10 \- $SECONDS)
    if [ $SLEEP -gt 0 ]; then
        echo "Sleeping for $SLEEP seconds"
        sleep $SLEEP
    fi
    SECONDS=0
    check_finished_files
    transfer_finished_files
done

vidprocman.sh

#!/bin/bash

#Video Processing Manager
#Script to manage the processing of video as it is received from the remote
#video capture source

#PIDFILE will be used to track background process ids
PIDFILE="pids.txt"
touch $PIDFILE
#WRAP will be the number at which filenames will wrap
WRAP=10000
#NAME_FMT defines the name format style, should be at least as many digits as WRAP
NAME_FMT="video%05d"
#FINISHED defines the path to a directory to store finished segments
FINISHED="finished"

shopt -s nullglob

#FFMPEG_FIFO will be used to communicate to background ffmpeg prcoess' stdin
if [ ! -p FFMPEG_FIFO ]; then
    mkfifo FFMPEG_FIFO
fi

#echo $$ > $PIDFILE

#This does nothing forever, but it keeps the fifo open
tail -f /dev/null > FFMPEG_FIFO &
echo $! >> $PIDFILE

function on_exit {
    #Runs when script ends for any reason
    echo "Cleaning up processes!"
    kill $CATPID
    echo 'q' > FFMPEG_FIFO
    kill $(cat $PIDFILE)
    rm FFMPEG_FIFO
    rm $PIDFILE
}
trap on_exit EXIT

function start_ffmpeg {
    echo "Starting video processing process..."
    #On a test machine, I have access to a VAAPI hardware decoder (https://trac.ffmpeg.org/wiki/Hardware/VAAPI)
    #in which case the first line would be:
    #ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -f h264 -i FFMPEG_FIFO
    # -hwaccel_output_format vaapi would be added if taking advantage of the h264_vaapi encoder
    #On a raspberry pi, I believe the line should be:
    #ffmpeg -f h264_mmal -i FFMPEG_FIFO
    ffmpeg -f h264 -i FFMPEG_FIFO \
    -filter_complex "[0]split=2[out1][tl],[tl]select='not(mod(n,25))',setpts=N/FRAME_RATE/TB[out2]" \
      -map "[out1]" -c:v libx264 -crf 23 -preset fast realtime.mkv \
      -map "[out2]" -c:v libx264 -crf 18 -preset veryslow timelapse.mkv &
    echo $! >> $PIDFILE
}

CUR=1
function pipe_finished_files {
    #echo "Entering pipe_finished_files"
    while true; do
        VIDFILE=$FINISHED/$(eval printf $NAME_FMT".h264" $CUR)
        #echo "Checking for $VIDFILE"
        if [ -f $VIDFILE ]; then
            CUR=$((CUR+1))
            if [ $CUR -gt $WRAP ]; then
                CUR=1
            fi
            #echo "cat $VIDFILE > FFMPEG_FIFO"
            cat $VIDFILE > FFMPEG_FIFO &
            CATPID=$!
            wait $CATPID
            rm $VIDFILE
        else
            break
        fi
    done
}

start_ffmpeg
SECONDS=0
while true; do
    #echo "Calling pipe_finished_files"
    pipe_finished_files
    #echo "Evaluation of sleep"
    SLEEP=$(expr 10 \- $SECONDS)
    if [ $SLEEP -gt 0 ]; then
        #echo "Sleeping for $SLEEP seconds"
        sleep $SLEEP
    fi
    SECONDS=0
done