Last active
February 4, 2022 22:17
-
-
Save polymonster/f7793c207d08f04db842fa94b6da46f7 to your computer and use it in GitHub Desktop.
A minimal example to show how to decode and extract frames using NDK AMediaCodec
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Deconding a h264 video stream... There are a few caveats which can cause some confusion. | |
// 1. 15 frames of 0 need flushing before we actually start pogressing. | |
// 2. Frames are extracted in chunks of 4 and appear in a swizzled order 4, 2, 1, 3, | |
// 3. The order seems consistent in blocks of 4, with the frames increasing from the middle entry rotating anti-clockwise. | |
// 4. Care needs to be taken when using frame numbers or timestamps to flag AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM | |
// Otherwise the decode may skip 4 frames (as the final frame will be delivered as the first in a block of 4). | |
// Havent experimented much yet but the blocks of 4 can also appear as blocks of 5, this may be due to frquency of keyframes | |
// tight loop will extract and decode all video frames | |
for(;;) { | |
// the -1 blocks, this is intended to be called on a background thread, waits until we have an available input buffer | |
s32 inputID = AMediaCodec_dequeueInputBuffer(m_codec, -1); | |
// extract data into the input buffer | |
size_t size = 0; | |
u8 *input = AMediaCodec_getInputBuffer(m_codec, inputID, &size); | |
auto sampleSize = AMediaExtractor_readSampleData(m_extractor, input, size); | |
// conversion timestamp to frame, if you need it | |
u64 sampleTimestamp = AMediaExtractor_getSampleTime(m_extractor); | |
u32 currentFrame = round(((sampleTimestamp * m_frameRate) / 1000.0f) / 1000.0f); | |
FW_LOG_CONSOLE("Extracted Sample Timestamp: %i : %i\n", sampleTimestamp, currentFrame); | |
// queue input buffer to be decoded | |
AMediaCodec_queueInputBuffer(m_codec, inputID, 0, sampleSize, sampleTimestamp, 0); | |
// check if a frame has been decoded and be dequed. | |
AMediaCodecBufferInfo info; | |
s32 outputID = AMediaCodec_dequeueOutputBuffer(m_codec, &info, 0); | |
if(outputID == AMEDIACODEC_INFO_TRY_AGAIN_LATER) { | |
// if this happens you may need to deque and extract more input frames, blocking on AMediaCodec_dequeueOutputBuffer | |
// can cause it to block indefinitely | |
continue; | |
} | |
else if (outputID > 0) { | |
// this will advance to the next frame, if you supply true on the end you need to render the frame | |
// on another thread via ASurfaceTexture_updateTexImage, if you supply false the frame is discarded. | |
// when seeking, you want to discard frames here by passing false. | |
AMediaCodec_releaseOutputBuffer(m_codec, outputID, false); | |
AMediaExtractor_advance(m_extractor); | |
} | |
} | |
// Example Output | |
/* | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 0 : 0 | |
Extracted Sample Timestamp: 166666 : 4 | |
Extracted Sample Timestamp: 83333 : 2 | |
Extracted Sample Timestamp: 41666 : 1 | |
Extracted Sample Timestamp: 125000 : 3 | |
Extracted Sample Timestamp: 333333 : 8 | |
Extracted Sample Timestamp: 250000 : 6 | |
Extracted Sample Timestamp: 208333 : 5 | |
Extracted Sample Timestamp: 291666 : 7 | |
Extracted Sample Timestamp: 500000 : 12 | |
Extracted Sample Timestamp: 416666 : 10 | |
Extracted Sample Timestamp: 375000 : 9 | |
Extracted Sample Timestamp: 458333 : 11 | |
Extracted Sample Timestamp: 666666 : 16 | |
Extracted Sample Timestamp: 583333 : 14 | |
Extracted Sample Timestamp: 541666 : 13 | |
Extracted Sample Timestamp: 625000 : 15 | |
Extracted Sample Timestamp: 833333 : 20 | |
Extracted Sample Timestamp: 750000 : 18 | |
Extracted Sample Timestamp: 708333 : 17 | |
Extracted Sample Timestamp: 791666 : 19 | |
Extracted Sample Timestamp: 1000000 : 24 | |
Extracted Sample Timestamp: 916666 : 22 | |
Extracted Sample Timestamp: 875000 : 21 | |
Extracted Sample Timestamp: 958333 : 23 | |
Extracted Sample Timestamp: 1166666 : 28 | |
Extracted Sample Timestamp: 1083333 : 26 | |
Extracted Sample Timestamp: 1083333 : 26 | |
Extracted Sample Timestamp: 1041666 : 25 | |
Extracted Sample Timestamp: 1125000 : 27 | |
Extracted Sample Timestamp: 1333333 : 32 | |
Extracted Sample Timestamp: 1250000 : 30 | |
Extracted Sample Timestamp: 1208333 : 29 | |
Extracted Sample Timestamp: 1291666 : 31 | |
Extracted Sample Timestamp: 1500000 : 36 | |
Extracted Sample Timestamp: 1416666 : 34 | |
Extracted Sample Timestamp: 1375000 : 33 | |
Extracted Sample Timestamp: 1458333 : 35 | |
Extracted Sample Timestamp: 1625000 : 39 | |
Extracted Sample Timestamp: 1583333 : 38 | |
Extracted Sample Timestamp: 1583333 : 38 | |
Extracted Sample Timestamp: 1541666 : 37 | |
Extracted Sample Timestamp: 1791666 : 43 | |
Extracted Sample Timestamp: 1708333 : 41 | |
Extracted Sample Timestamp: 1666666 : 40 | |
Extracted Sample Timestamp: 1750000 : 42 | |
Extracted Sample Timestamp: 2000000 : 48 | |
Extracted Sample Timestamp: 1916666 : 46 | |
Extracted Sample Timestamp: 1833333 : 44 | |
Extracted Sample Timestamp: 1875000 : 45 | |
Extracted Sample Timestamp: 1958333 : 47 | |
Extracted Sample Timestamp: 2208333 : 53 | |
Extracted Sample Timestamp: 2125000 : 51 | |
Extracted Sample Timestamp: 2041666 : 49 | |
Extracted Sample Timestamp: 2083333 : 50 | |
Extracted Sample Timestamp: 2166666 : 52 | |
Extracted Sample Timestamp: 2416666 : 58 | |
Extracted Sample Timestamp: 2333333 : 56 | |
Extracted Sample Timestamp: 2250000 : 54 | |
Extracted Sample Timestamp: 2291666 : 55 | |
Extracted Sample Timestamp: 2375000 : 57 | |
Extracted Sample Timestamp: 2625000 : 63 | |
Extracted Sample Timestamp: 2541666 : 61 | |
Extracted Sample Timestamp: 2458333 : 59 | |
Extracted Sample Timestamp: 2500000 : 60 | |
Extracted Sample Timestamp: 2583333 : 62 | |
Extracted Sample Timestamp: 2833333 : 68 | |
Extracted Sample Timestamp: 2750000 : 66 | |
Extracted Sample Timestamp: 2666666 : 64 | |
Extracted Sample Timestamp: 2708333 : 65 | |
Extracted Sample Timestamp: 2791666 : 67 | |
Extracted Sample Timestamp: 3000000 : 72 | |
Extracted Sample Timestamp: 2916666 : 70 | |
Extracted Sample Timestamp: 2875000 : 69 | |
Extracted Sample Timestamp: 2958333 : 71 | |
Extracted Sample Timestamp: 3083333 : 74 | |
Extracted Sample Timestamp: 3041666 : 73 | |
Extracted Sample Timestamp: 3291666 : 79 | |
Extracted Sample Timestamp: 3208333 : 77 | |
Extracted Sample Timestamp: 3125000 : 75 | |
Extracted Sample Timestamp: 3166666 : 76 | |
Extracted Sample Timestamp: 3250000 : 78 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
Extracted Sample Timestamp: -1 : -1 | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment