Пример построения векторов движения с помощью libav. Выводит координаты точек и векторов на stdout. Проверялось на коротких mp4 (h264 / aac). Начальный вариант был взят тут: http://www.princeton.edu/~jiasic/cos435/motion_vector.c и переписан под современный ffmpeg.

motion_vector.c

/**
 * MOTION_VECTOR
 *
 *      Calculates motion vector using ffmpeg and libavcodec.
 *
 * @authors First modifications by Victor Hsieh.
 *          Second modifications by Jiasi Chen.
 *          Third modifications by Ilya Nikitin <w@w-495.ru>.
 *
 * @date:   2014.05.10 23:30:13
 * @version it works with:
 *              ffmpeg version N-50086-gd41efc1
 *              built on May 10 2014 03:28:08 with gcc 4.8.2
 *                  (GCC) 20131212 (Red Hat 4.8.2-7)
 *              configuration: --prefix=/home/w495/ffmpeg_build
 *              --extra-libs=-ldl --enable-gpl --enable-libass
 *              --enable-libfdk-aac --enable-libfreetype --enable-libmp3lame
 *              --enable-libopus --enable-libtheora --enable-libvorbis
 *              --enable-libvpx --enable-libx264 --enable-nonfree
 *              --enable-x11grab
 *
 *              libavutil      52. 17.102 / 52. 17.102
 *              libavcodec     54. 92.100 / 54. 92.100
 *              libavformat    54. 63.100 / 54. 63.100
 *              libavdevice    54.  3.103 / 54.  3.103
 *              libavfilter     3. 38.103 /  3. 38.103
 *              libswscale      2.  2.100 /  2.  2.100
 *              libswresample   0. 17.102 /  0. 17.102
 *              libpostproc    52.  2.100 / 52.  2.100
**/

#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <math.h>

#define IS_INTERLACED(a) ((a)&MB_TYPE_INTERLACED)
#define IS_16X16(a)      ((a)&MB_TYPE_16x16)
#define IS_16X8(a)       ((a)&MB_TYPE_16x8)
#define IS_8X16(a)       ((a)&MB_TYPE_8x16)
#define IS_8X8(a)        ((a)&MB_TYPE_8x8)
#define USES_LIST(a, list) ((a) & ((MB_TYPE_P0L0|MB_TYPE_P1L0)<<(2*(list))))

#define     FF_I_TYPE   1
#define     FF_P_TYPE   2
#define     FF_B_TYPE   3
#define     FF_S_TYPE   4

#define NO_MV_MAGIC 16384

#define LOG_HELPER(fmt, ...)                    \
    fprintf(                                    \
        stderr,                                 \
        "\033[32mLOG:\033[0m "                  \
        "\033[33m%s\033[0m "                    \
        "\033[36m%s\033[0m "                    \
        "[\033[1m%d\033[0m] : " fmt "%s",       \
        __FILE__,                               \
        __FUNCTION__,                           \
        __LINE__,                               \
        __VA_ARGS__                             \
    )

#define LOG(...) LOG_HELPER(__VA_ARGS__, "\n")


int count = 0;
double sum = 0;
double h;
double w;

bool get_next_frame(AVFormatContext*, AVCodecContext*, int, AVFrame*);

void print_mv_matrix(int index, AVFrame * pict, AVCodecContext * dec_ctx);

void print_vector(int x, int y, int dx, int dy);


int main(int argc, char *argv[]) {
    AVFormatContext *fmt_ctx;
    AVCodecContext *dec_ctx;
    AVCodec *dec;
    AVFrame *frame;
    int stream;
    int i;
    int f = 1;

    if (argc != 4) {
        fprintf(stderr, "Usage: %s file h w | %d \n", argv[0], argc);
        exit(1);
    }

    /* Save height and width */

    h = strtod(argv[2], NULL);
    w = strtod(argv[3], NULL);

    /*  Register all formats and codecs */

    av_register_all();

    /*  Open video file */
    if (avformat_open_input(&fmt_ctx, argv[1], NULL, NULL) != 0){
        perror(argv[1]);
        av_log(NULL, AV_LOG_ERROR, "Couldn't open file\n");
        exit(1);
    }

    /* Retrieve stream information */
    if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {
        av_log(NULL, AV_LOG_ERROR, "Couldn't find stream information\n");
        exit(1);
    }

    /* Dump information about file onto standard error */
    av_dump_format(fmt_ctx, 0, argv[1], false);

    /* Find the first video stream */
    stream = -1;
    for (i = 0; i < fmt_ctx->nb_streams; i++) {
        AVCodecContext *cc = fmt_ctx->streams[i]->codec;

        if (cc->codec_type == AVMEDIA_TYPE_VIDEO) {
            /* don't care FF_DEBUG_VIS_MV_B_BACK */
            cc->debug_mv = FF_DEBUG_VIS_MV_P_FOR | FF_DEBUG_VIS_MV_B_FOR;
            stream = i;
            break;
        }
    }

    if (stream == -1){
        av_log(NULL, AV_LOG_ERROR, "Didn't find a video stream\n");
        exit(1);
    }

    /* Get a pointer to the codec context for the video stream */
    dec_ctx = fmt_ctx->streams[stream]->codec;

    /* Find the decoder for the video stream */
    dec = avcodec_find_decoder(dec_ctx->codec_id);
    if (dec == NULL){
        av_log(NULL, AV_LOG_ERROR, "Codec not found\n");
        exit(1);
    }

    /* Inform the codec that we can handle truncated bitstreams
     * -- i.e., bitstreams where frame boundaries can fall
     * in the middle of packets
    */
    if (dec->capabilities & CODEC_CAP_TRUNCATED)
        dec_ctx->flags |= CODEC_FLAG_TRUNCATED;

    /* Open codec */
    if (avcodec_open2(dec_ctx, dec, NULL) < 0) {
        av_log(NULL, AV_LOG_ERROR, "Could not open codec\n");
        exit(1);
    }

    /* Hack to correct wrong frame rates
     * that seem to be generated
     * by some codecs:
     *      if (dec_ctx->frame_rate>1000 && dec_ctx->frame_rate_base==1)
     *          dec_ctx->frame_rate_base=1000;
     */

    /* Allocate video frame */
    frame = avcodec_alloc_frame();
    while (get_next_frame(fmt_ctx, dec_ctx, stream, frame)) {
        /* Ignore I-Frame */
        if (frame->pict_type != FF_I_TYPE) {
            print_mv_matrix(f, frame, dec_ctx);
        }
        ++f;
    }

    av_free(frame);
    avcodec_close(dec_ctx);
    avformat_close_input(&fmt_ctx);

    printf("sum mv: %f, total # mv: %d\n",sum,count);
    printf("%f\n", sum / (double) count);

    return 0;
}

bool get_next_frame(
    AVFormatContext *fmt_ctx,
    AVCodecContext  *dec_ctx,
    int stream,
    AVFrame *frame
) {
    static AVPacket packet;
    int bytesDecoded;
    int frameFinished;

    /* read all packets */
    while (1) {
        if ((bytesDecoded = av_read_frame(fmt_ctx, &packet)) < 0)
            break;

        if (packet.stream_index == stream) {
            avcodec_get_frame_defaults(frame);
            frameFinished = 0;
            bytesDecoded = avcodec_decode_video2(
                dec_ctx,
                frame,
                &frameFinished,
                &packet
            );
            if (bytesDecoded < 0) {
                av_log(NULL, AV_LOG_ERROR, "Error decoding video\n");
                break;
            }
            if (frameFinished) {
                frame->pts = av_frame_get_best_effort_timestamp(frame);
                break;
            }
        }
        av_free_packet(&packet);
    }
    return frameFinished != 0;
}

void print_vector(int x, int y, int dx, int dy) {
    if (dx != NO_MV_MAGIC && dy != NO_MV_MAGIC) {
        sum =
            sum + sqrt((double) dx * (double) dx / w / w +
                       (double) dy * (double) dy / h / h);
        count++;
    }
    printf("%d %d ; %d %d\n", x, y, dx, dy);
}

/**
 * @fn  Print motion vector for each macroblock in this frame.
 *      If there is no motion vector in some macroblock,
 *      it prints a magic number NO_MV_MAGIC.
**/
void print_mv_matrix(int index, AVFrame * pict, AVCodecContext * dec_ctx) {
    const int mb_width = (dec_ctx->width + 15) / 16;
    const int mb_height = (dec_ctx->height + 15) / 16;
    const int mb_stride = mb_width + 1;
    const int mv_sample_log2 = 4 - pict->motion_subsample_log2;
    const int mv_stride =
        (mb_width << mv_sample_log2)
            + (dec_ctx->codec_id == CODEC_ID_H264 ? 0 : 1);
    const int quarter_sample = (dec_ctx->flags & CODEC_FLAG_QPEL) != 0;
    const int shift = 1 + quarter_sample;


    printf("frame %d, %d x %d\n", index, mb_height, mb_width);

    int mb_y;
    for (mb_y = 0; mb_y < mb_height; mb_y++) {
        int mb_x;
        for (mb_x = 0; mb_x < mb_width; mb_x++) {
            const int mb_index = mb_x + mb_y * mb_stride;

            if (pict->motion_val) {
                int type;
                for (type = 0; type < 3; type++) {
                    int direction = 0;

                    switch (type) {
                        case 0:
                            if (pict->pict_type != FF_P_TYPE)
                                continue;
                            direction = 0;
                            break;
                        case 1:
                            if (pict->pict_type != FF_B_TYPE)
                                continue;
                            direction = 0;
                            break;
                        case 2:
                            if (pict->pict_type != FF_B_TYPE)
                                continue;
                            direction = 1;
                            break;
                    }

                    if (!USES_LIST(pict->mb_type[mb_index], direction)) {

                        if (IS_8X8(pict->mb_type[mb_index])) {
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                        }
                        else if (IS_16X8(pict->mb_type[mb_index])) {
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                        }
                        else if (IS_8X16(pict->mb_type[mb_index])) {
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                        }
                        else {
                            print_vector(mb_x, mb_y, NO_MV_MAGIC, NO_MV_MAGIC);
                        }
                        continue;
                    }

                    if (IS_8X8(pict->mb_type[mb_index])) {
                        int i = 0;
                        for (i = 0; i < 4; i++) {
                            int xy =
                                (mb_x * 2 + (i & 1) +
                                 (mb_y * 2 + (i >> 1)) * mv_stride)
                                    << (mv_sample_log2 -1);
                            int dx =
                                (pict->motion_val[direction][xy][0] >> shift);
                            int dy =
                                (pict->motion_val[direction][xy][1] >> shift);
                            print_vector(mb_x, mb_y, dx, dy);
                        }
                    }
                    else if (IS_16X8(pict->mb_type[mb_index])) {
                        int i = 0;
                        for (i = 0; i < 2; i++) {
                            int xy =
                                (mb_x * 2 +
                                 (mb_y * 2 +
                                  i) * mv_stride) << (mv_sample_log2 - 1);
                            int dx =
                                (pict->motion_val[direction][xy][0] >> shift);
                            int dy =
                                (pict->motion_val[direction][xy][1] >> shift);

                            if (IS_INTERLACED(pict->mb_type[mb_index]))
                                dy *= 2;

                            print_vector(mb_x, mb_y, dx, dy);
                        }
                    }
                    else if (IS_8X16(pict->mb_type[mb_index])) {
                        int i = 0;
                        for (i = 0; i < 2; i++) {
                            int xy =
                                (mb_x * 2 + i +
                                 mb_y * 2 * mv_stride)
                                    << (mv_sample_log2 - 1);
                            int dx =
                                (pict->motion_val[direction][xy][0] >> shift);
                            int dy =
                                (pict->motion_val[direction][xy][1] >> shift);
                            if (IS_INTERLACED(pict->mb_type[mb_index]))
                                dy *= 2;
                            print_vector(mb_x, mb_y, dx, dy);
                        }
                    }
                    else {
                        int xy = (mb_x + mb_y * mv_stride) << mv_sample_log2;
                        int dx =
                            (pict->motion_val[direction][xy][0] >> shift);
                        int dy =
                            (pict->motion_val[direction][xy][1] >> shift);
                        print_vector(mb_x, mb_y, dx, dy);
                    }
                }
            }
            printf("--\n");
        }
        printf("====\n");
    }
}

Теоретические изыскания по теме:
http://www.princeton.edu/~jiasic/cos435/report.pdf