Script to encode videos from my video camera. Runs multiple ffmpeg processes to optimize CPU usage rather than relying on one ffmpeg process to use multiple processors.

.gitignore

__pycache__/

encode_videos.py

#!/usr/bin/env python3
"""
Simple video conversion management script
"""

import argparse
import datetime
import os, subprocess, sys, time
from os.path import join, splitext
import queue
import shutil
import threading
import tempfile

from settings import ALL_PARAMS

job_queue = queue.Queue()

NUM_JOBS=8
NICE_LEVEL=0

def worker():
    while True:
        args, kwargs = job_queue.get()
        encode_file(*args, **kwargs)
        job_queue.task_done()

def queue_encode_file(*args, **kwargs):
    job_queue.put((args, kwargs))

def encode_file(input_file, chosen_config, params, output_base_dir, force=False, label=None, subdir=None):
    """
    Encode a file given some params

    params is a dictionary that must contain the following keys:
        -ext: the extension for the output file
        -num_passes: number of passes for encoder
        -encoder: path to the encoder binary to use
        -pass_specific: a tuple of pass specific options
    """
    print("***** Input file:", input_file)
    input_file_dir = os.path.split(input_file)[0]
    if label is None:
        outputdir = join(output_base_dir, chosen_config)
    else:
        outputdir = join(output_base_dir, label)
    if subdir:
        outputdir = join(outputdir, subdir)
    if not os.path.isdir(outputdir):
        try:
            os.makedirs(outputdir)
        except:
            print("Failed to create", outputdir, "probably because another thread created it")
    output_file_name = splitext(os.path.basename(input_file))[0] + params['ext']
    real_output_file = join(outputdir, output_file_name)
    temp_dir = tempfile.mkdtemp()
    tmp_output_file = join(temp_dir, output_file_name)
    print("***** Output file:", real_output_file)
    print("***** Temp output file:", tmp_output_file)
    if os.path.isfile(real_output_file) and not force:
        print("Skipping %s" % (input_file))
        return

    try:
        for i in range(1, params['num_passes']+1):
            command = "{encoder} " \
                      "{options} " \
                      "{pass_specific}" \
                      .format(encoder=params['encoder'],
                              options=params['options'],
                              pass_specific=params['pass_specific'][i-1])
            command = command.format(input_file=input_file,
                                     output_file=tmp_output_file)
            print("***** Running the following command: %s" % (command))
            start_time = time.time()
            proc = subprocess.Popen(command.split())
            retcode = proc.wait()
            if retcode < 0:
                print("***** Error calling encoder (retcode: %s)" % retcode)
                raise Exception("Failed calling encoder")
            else:
                print("***** Success calling encoder for pass %d/%d" % (i, params['num_passes']))
            print("Took %s s" % (round(time.time() - start_time, 2),))

        if retcode == 0:
            shutil.move(tmp_output_file, real_output_file)
    finally:
        if os.path.exists(temp_dir):
            shutil.rmtree(temp_dir)

def parse_args():
    parser = argparse.ArgumentParser(description='Encode some videos.')
    parser.add_argument('config', metavar='CONFIG', type=check_config,
                        help='an encoding configuration')
    parser.add_argument('video_directory', metavar='DIR', type=str,
                        help='directory containing videos')
    parser.add_argument('pattern', metavar='PATTERN', type=str,
                        help='PATTERN of video to encode (ie. .MTS)')
    parser.add_argument('output_base_dir', metavar='OUTPUT',
                        help='base output directory')
    parser.add_argument('--force', '-f', action='store_true', default=False,
                        help='Force re-encode even if destination file already exists')
    parser.add_argument('--label', '-l', help='alternative label to use for output directory')
    args = parser.parse_args()
    return args

def check_config(config):
    if config not in ALL_PARAMS:
        print("Invalid config: {0}." \
              "The available configurations are:".format(config))
        for config in list(ALL_PARAMS.keys()):
            print(" "*4 + config)
        sys.exit(1)
    return config

def encode_videos(video_directory, pattern, config, output_base_dir, force=False, label=None):
    # Start up some worker threads
    for i in range(NUM_JOBS):
        t = threading.Thread(target=worker)
        t.daemon = True
        t.start()

    if not os.path.isdir(config):
        os.makedirs(config)
    params = ALL_PARAMS[config]

    video_paths = []
    for root, dirs, files in os.walk(video_directory):
        for name in files:
            if pattern.lower() == os.path.splitext(name)[1].lower():
                video_paths.append(os.path.join(root, name))

    for input_file in video_paths:
        mtime = datetime.datetime.fromtimestamp(os.path.getmtime(input_file))
        year = str(mtime.year)
        queue_encode_file(input_file, config, params, output_base_dir, force=force, label=label, subdir=year)

    job_queue.join()

def main():
    """
    Main script
    """
    args = parse_args()

    encode_videos(args.video_directory, args.pattern, args.config, args.output_base_dir, force=args.force,
                 label=args.label)

if __name__ == '__main__':
    main()

settings.py

ALL_PARAMS = {}

ALL_PARAMS['x264_test'] = {
        'encoder':  '/usr/local/bin/ffmpeg', \
        'options':  '-i {input_file} ' \
                    '-threads 1 ' \
                    '-y ' \
                    '-c:a ac3 ' \
                    '-c:v libx264 ' \
                    '-preset veryfast ' \
                    '-tune film ' \
                    '-profile:v baseline -level 3.0 ' \
                    '-crf 28 ' \
                    '-vf yadif=0 ' \
                    '-r 30000/1001 ' \
                    '-s 1280x720 ' \
                    '-movflags +faststart ' \
                    '{output_file}',
        'ext': '.mp4',
        'pass_specific': ( ('',) ),
        'num_passes': 1,
        }


ALL_PARAMS['x264_60p'] = {
        'encoder':  '/usr/bin/ffmpeg', \
        'options':  '-r 30000/1001 ' \
                    ' -i {input_file} ' \
                    '-threads 1 ' \
                    '-y ' \
                    '-c:a libfdk_aac -b:a 128k ' \
                    '-sws_flags lanczos ' \
                    '-c:v libx264 ' \
                    '-preset medium ' \
                    '-tune film ' \
                    '-profile:v high -level 4.1 ' \
                    '-crf 23 ' \
                    '-x264opts bframes=2:keyint=30:min-keyint=30 ' \
                    '-vf yadif=1:0,mcdeint=fast:0:1 ' \
                    '-r 60000/1001 ' \
                    '-s 1280x720 ' \
                    '-g 120 ' \
                    '-movflags +faststart ' \
                    '{output_file}',
        'ext': '.mp4',
        'pass_specific': ( ('',) ),
        'num_passes': 1,
        }

ALL_PARAMS['x264_30p'] = {
        'encoder':  '/usr/bin/ffmpeg', \
        'options':  '-r 30000/1001 ' \
                    ' -i {input_file} ' \
                    '-threads 1 ' \
                    '-y ' \
                    '-c:a libfdk_aac -b:a 128k ' \
                    '-c:v libx264 ' \
                    '-preset veryfast ' \
                    '-tune film ' \
                    '-profile:v high -level 4.1 ' \
                    '-bf 2 ' \
                    '-crf 23 ' \
                    '-vf yadif ' \
                    '-r 30000/1001 ' \
                    '-g 15 ' \
                    '-movflags +faststart ' \
                    '{output_file}',
        'ext': '.mp4',
        'pass_specific': ( ('',) ),
        'num_passes': 1,
        }