first experiment at using vaapi

vaapi_crap.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <va/va.h>
#include <va/va_enc_h264.h>
#include <va/va_x11.h>

/*
 * http://www.intel.cn/content/dam/www/public/us/en/documents/white-papers/atom-e6xx-video-encoding-paper.pdf
 * This paper is crap. Since 2011 almost all VA API was changed
 */

#define CHECK_VASTATUS(va_status,func)                                  \
    if (va_status != VA_STATUS_SUCCESS) {                               \
        fprintf(stderr,"%s:%s (%d) failed,exit\n", __func__, func, __LINE__); \
        exit(1);                                                        \
    }


typedef struct xxx_vaapi_ctx {
        Display *display;
        VADisplay va_display;
        int va_version_major;
        int va_version_minor;

        unsigned va_format;
} xxx_vaapi_ctx;

static xxx_vaapi_ctx *xxx_vaapi_open(Display *display) {
        int opened_display = 0;
        xxx_vaapi_ctx *ctx = 0;
        VADisplay *va_display = NULL;
        VAStatus va_status;
        int major = 0, minor = 0;
        ctx = malloc(sizeof(xxx_vaapi_ctx));

        if (!ctx) {
                goto err;
        }
        memset(ctx, 0, sizeof(xxx_vaapi_ctx));
        
        if (!display) {
                display = XOpenDisplay(NULL);
                if (display) {
                        opened_display = 1;
                }
                else {
                        goto err;
                }
        }

        va_display = vaGetDisplay(display);
        if (!va_display || !vaDisplayIsValid(va_display)) {
                goto err;
        }

        va_status = vaInitialize(va_display, &major, &minor);
        if (va_status != VA_STATUS_SUCCESS) {
                goto err;
        }

        if (opened_display) {
                ctx->display = display;
        }

        ctx->va_display = va_display;
        ctx->va_version_major = major;
        ctx->va_version_minor = minor;

        return ctx;      
err:
        if (va_display) {
                vaTerminate(va_display);
        }
        if (display && opened_display) {
                XCloseDisplay(display);
        }
        if (ctx) {
                free(ctx);
        }
        return NULL;
}

static void xxx_vaapi_close(xxx_vaapi_ctx *ctx) {
        if (!ctx) {
                return;
        }

        if (ctx->va_display) {
                vaTerminate(ctx->va_display);
        }
        if (ctx->display) {
                XCloseDisplay(ctx->display);
        }
        free(ctx);
}

enum {
        N_SURFACES = 3,
        FRAME_W = 1920,
        FRAME_H = 1080,

        FRAME_COUNT = 1,
        FRAME_RATE = 24,
        INTRA_PERIOD = FRAME_COUNT,
};

static void test_encoder(xxx_vaapi_ctx *ctx) {
        int i;
        VAStatus va_status;
        VAConfigAttrib attrib[2];
        attrib[0].type = VAConfigAttribRTFormat;
        attrib[0].value = VA_RT_FORMAT_YUV420;
        attrib[1].type = VAConfigAttribRateControl;
        attrib[1].value = VA_RC_VBR;

        VAConfigID config;
        va_status = vaCreateConfig(ctx->va_display, VAProfileH264High,
                VAEntrypointEncSlice, attrib, 2, &config);
        CHECK_VASTATUS(va_status, "vaCreateConfig");

        VASurfaceAttrib attribs[] = {
                {
                        .type = VASurfaceAttribPixelFormat,
                        .flags = VA_SURFACE_ATTRIB_SETTABLE,
                        .value = {
                                .type = VAGenericValueTypeInteger,
                                .value.i = VA_FOURCC_NV12, 
                        }
                }
        };

        VASurfaceID surfaces[N_SURFACES];
        va_status = vaCreateSurfaces(ctx->va_display,
                VA_RT_FORMAT_YUV420,
                FRAME_W, FRAME_H,
                surfaces, N_SURFACES,
                attribs, 1);
        CHECK_VASTATUS(va_status, "vaCreateSurfaces");

        VAContextID context;
        va_status = vaCreateContext(ctx->va_display, config,
                FRAME_W, FRAME_H, VA_PROGRESSIVE, 0, 0, &context);
        CHECK_VASTATUS(va_status, "vaCreateContext");

        VABufferID coded_buf[N_SURFACES];
        for (i = 0; i < N_SURFACES; i++) {
                size_t buf_size = (FRAME_W * FRAME_H * 400) / (16 * 16);
                va_status = vaCreateBuffer(ctx->va_display, context, VAEncCodedBufferType,
                        buf_size, 1, NULL, &coded_buf[i]);
                CHECK_VASTATUS(va_status, "vaCreateBuffer");
        }

        VAImage image;
        void *pbuffer = NULL;
        vaDeriveImage(ctx->va_display, surfaces[0], &image);
        vaMapBuffer(ctx->va_display, image.buf, &pbuffer);

        int frame = 0;
        while (frame < FRAME_COUNT) {
                puts("frame");
                va_status = vaBeginPicture(ctx->va_display, context, surfaces[0]);
                CHECK_VASTATUS(va_status, "vaBeginPicture");
                
                if (frame == 0) {
                        puts("sequence params");
                        VABufferID seq_buf_id;
                        VAEncSequenceParameterBufferH264 seq_h264 = {
                                .level_idc = 41, /* SH_LEVEL_3 */
                                .picture_width_in_mbs = FRAME_W / 16,
                                .picture_height_in_mbs = FRAME_H / 16,
                                .bits_per_second = 14 * 1000 * 1000,
                                .intra_period = INTRA_PERIOD,
                                .seq_fields = {
                                        .bits = {
                                                .frame_mbs_only_flag = 1,
                                        },
                                },
                        };

                        va_status = vaCreateBuffer(ctx->va_display, context,
                                VAEncSequenceParameterBufferType,
                                sizeof(seq_h264), 1, &seq_h264, &seq_buf_id);
                        CHECK_VASTATUS(va_status, "vaCreateBuffer");
                        
                        va_status = vaRenderPicture(ctx->va_display, context, &seq_buf_id, 1);
                        CHECK_VASTATUS(va_status, "vaRenderPicture");
                }

                VAEncPictureParameterBufferH264 pic_h264 = {
                        .CurrPic = {
                                .picture_id = surfaces[0],
                        },
                        .ReferenceFrames = {
                                [0] = {
                                        .picture_id = surfaces[0],
                                },
                                [1] = {
                                        .flags = VA_PICTURE_H264_INVALID,
                                        .picture_id = VA_INVALID_SURFACE,
                                },
                        },
                        .pic_fields = {
                                .bits = {
                                        .reference_pic_flag = 1,
                                },
                        },
                        .coded_buf = coded_buf[0],
                        .last_picture = 1,
                };
                
                puts("picture params");
                VABufferID pic_param_buf;
                va_status = vaCreateBuffer(ctx->va_display, context, VAEncPictureParameterBufferType,
                        sizeof(pic_h264), 1, &pic_h264, &pic_param_buf);
                CHECK_VASTATUS(va_status, "vaCreateBuffer");
                va_status = vaRenderPicture(ctx->va_display, context, &pic_param_buf, 1);
                CHECK_VASTATUS(va_status, "vaRenderPicture");

                puts("slice params");
                VAEncSliceParameterBufferH264 slice_h264 = {
                        .num_macroblocks = ((FRAME_W + 15) & ~15)* ((FRAME_H + 15) & ~15) / (16 * 16),
                        .slice_type = 2,
                };
                VABufferID slice_param_buf;
                va_status = vaCreateBuffer(ctx->va_display, context, VAEncSliceParameterBufferType,
                        sizeof(slice_h264), 1, &slice_h264, &slice_param_buf);
                CHECK_VASTATUS(va_status, "vaCreateBuffer");
                va_status = vaRenderPicture(ctx->va_display, context, &slice_param_buf, 1);
                CHECK_VASTATUS(va_status, "vaRenderPicture");

                va_status = vaEndPicture(ctx->va_display, context);
                CHECK_VASTATUS(va_status, "vaEndPicture");

                va_status = vaSyncSurface(ctx->va_display, surfaces[0]); 
                CHECK_VASTATUS(va_status, "vaSyncSurface");

                frame++;
        }

        void *output = NULL;
        va_status = vaMapBuffer(ctx->va_display, coded_buf[0], &output);
        CHECK_VASTATUS(va_status, "vaMapBuffer");
        unsigned long out_sz = *((unsigned long*)output);
        unsigned long out_off = *((unsigned long*)(output + 4));
        printf("output size=%lx offset=%lx\n", out_sz, out_off);

        FILE *fout;
        fout = fopen("out.h264", "wb");
        if (fout != NULL) {
                fwrite(output + out_off, out_sz, 1, fout);
                fflush(fout);
                fclose(fout);
        }

        vaUnmapBuffer(ctx->va_display, coded_buf[0]);

        vaUnmapBuffer(ctx->va_display, image.buf);
        vaDestroyImage(ctx->va_display, image.image_id);

        vaDestroyConfig(ctx->va_display, config);
}

int main() {
        xxx_vaapi_ctx *ctx = xxx_vaapi_open(NULL);
        if (ctx) {
                printf("opened vaapi version %d.%d\n",
                        ctx->va_version_major, ctx->va_version_minor);
                test_encoder(ctx);
        }
        else {
                printf("failed to open vaapi\n");
        }

        xxx_vaapi_close(ctx);
}