RTP 시간 사용, FFmpeg 인코더로 인코딩, AVFilter 사용하여 리셈플링, ADPCM 프레임 크기 적용

use-ffmpeg-adpcm-resampler-framesize.patch

diff --git a/edc-codec.c b/edc-codec.c
index b4aa4278..206c0d46 100644
--- a/edc-codec.c
+++ b/edc-codec.c
@@ -579,7 +579,7 @@ edc_codec_new (EdcCodecID    codec_id,
   else
     codec->context_name = edc_strdup (codec->is_video ? "video" : "audio");
 
-  if (codec_id == EDC_CODEC_ADPCM)
+  if (0 && codec_id == EDC_CODEC_ADPCM)
     {
       codec->adpcm = edc_adpcm_new ();
       if (!codec->adpcm)
@@ -911,7 +911,7 @@ edc_codec_decode_audio (EdcCodec    *codec,
   if (!(src_len > 0))
     return -1;
 
-  if (codec->codec_id == EDC_CODEC_ADPCM)
+  if (0 && codec->codec_id == EDC_CODEC_ADPCM)
     {
       *dest_len = edc_adpcm_decode (codec->adpcm,
                                     (unsigned char *) src,
@@ -986,7 +986,7 @@ edc_codec_decode_audio2 (EdcCodec   *codec,
   if (src_len <= 0)
     return NULL;
 
-  if (codec->codec_id == EDC_CODEC_ADPCM)
+  if (0 && codec->codec_id == EDC_CODEC_ADPCM)
     {
       int len;
 
@@ -1070,7 +1070,7 @@ edc_codec_encode (EdcCodec     *codec,
       return -1;
     }
 
-  if (codec->codec_id == EDC_CODEC_ADPCM)
+  if (0 && codec->codec_id == EDC_CODEC_ADPCM)
     {
       *dest_len = edc_adpcm_encode (codec->adpcm,
                                     (short *) src,
diff --git a/edc-rtsp.c b/edc-rtsp.c
index 161ca75d..2dbb7345 100644
--- a/edc-rtsp.c
+++ b/edc-rtsp.c
@@ -20,6 +20,11 @@
 #include <libswresample/swresample.h>
 #include <libavutil/random_seed.h>
 #include <libavutil/intreadwrite.h>
+#include <libavutil/timestamp.h>
+#include <libavfilter/avfilter.h>
+#include <libavfilter/buffersrc.h>
+#include <libavfilter/buffersink.h>
+#include <libavutil/timestamp.h>
 
 #ifndef _
 #if defined(linux) && defined(PACKAGE)
@@ -65,12 +70,16 @@ struct _EdcRtspConvert
 
   SwrContext         *resampler;
 
-  EdcAdpcm           *encoder;
+  AVCodecContext     *encoder;
   EdcCodecID          output_codec_id;
   enum AVSampleFormat output_sample_fmt;
   int                output_bytes_per_sample;
   int                output_sample_rate;
   int                output_channels;
+
+  AVFilterGraph   *graph;
+  AVFilterContext *src;
+  AVFilterContext *sink;
 };
 
 typedef enum _EdcRtspSourceType
@@ -109,7 +118,6 @@ struct _EdcRtspAudioOut
   uint32_t    ssrc;
 };
 
-
 /**
  * EdcRtsp object
  */
@@ -120,6 +128,8 @@ struct _EdcRtsp
   EdcRtspFrameRecv *frame_recv;
   EdcRtspErrorFunc  err_func;
 
+  int64_t base_timestamp;
+
   struct
   {
     EdcRtspCheckExitFunc func;
@@ -173,6 +183,8 @@ struct _EdcRtsp
     int        width;
     int        height;
 
+    int64_t start_pts;
+
     EdcBuf *(*make_frame) (EdcRtsp  *rtsp,
                            AVPacket *pkt);
   } video;
@@ -182,6 +194,8 @@ struct _EdcRtsp
     int        has_stream;
     EdcCodecID codec_id;
 
+    int64_t start_pts;
+
     EdcBuf *(*make_frame) (EdcRtsp  *rtsp,
                            AVPacket *pkt);
     /**
@@ -263,6 +277,171 @@ edc_rtsp_log (EdcRtsp     *rtsp,
            rtsp->transport);
 }
 
+static int
+mp4serv_init_audio_resampling_filter (EdcRtsp *rtsp)
+{
+  EdcRtspConvert *c;
+  const AVFilter *buffersrc;
+  const AVFilter *buffersink;
+  AVFilterInOut *outputs;
+  AVFilterInOut *inputs;
+  char args[512];
+  int ret;
+
+  c = &rtsp->audio.convert;
+
+  c->graph = avfilter_graph_alloc ();
+  if (!c->graph)
+    {
+      edc_rtsp_debug (rtsp, "%s", "failed to create audio filter graph");
+      return -1;
+    }
+
+  buffersrc = avfilter_get_by_name ("abuffer");
+  if (!buffersrc)
+    {
+      edc_rtsp_debug (rtsp, "failed to get `%s` filter", "abuffer");
+      return -1;
+    }
+
+  g_snprintf (args, sizeof(args),
+              "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%"PRIx64,
+              1, c->input_sample_rate,
+              c->input_sample_rate,
+              av_get_sample_fmt_name (c->input_sample_fmt),
+              av_get_default_channel_layout (c->input_channels));
+
+  ret = avfilter_graph_create_filter (&c->src,
+                                      buffersrc,
+                                      "in",
+                                      args,
+                                      NULL,
+                                      c->graph);
+  if (ret < 0)
+    {
+      edc_rtsp_debug (rtsp, "failed to create `%s` filter", "abuffer");
+      return -1;
+    }
+
+  buffersink = avfilter_get_by_name ("abuffersink");
+  if (!buffersink)
+    {
+      edc_rtsp_debug (rtsp, "failed to get `%s` filter", "abuffersink");
+      return -1;
+    }
+
+  ret = avfilter_graph_create_filter (&c->sink,
+                                      buffersink,
+                                      "out",
+                                      NULL,
+                                      NULL,
+                                      c->graph);
+  if (ret < 0)
+    {
+      edc_rtsp_debug (rtsp, "failed to create `%s` filter", "abuffersink");
+      return -1;
+    }
+
+  ret = av_opt_set_bin (c->sink,
+                        "sample_fmts",
+                        (uint8_t *) &c->output_sample_fmt,
+                        sizeof (c->output_sample_fmt),
+                        AV_OPT_SEARCH_CHILDREN);
+  if (ret < 0)
+    {
+      edc_rtsp_debug (rtsp, "%s", "failed to set output sample format");
+      return -1;
+    }
+
+  {
+    int64_t channel_layout;
+
+    channel_layout = av_get_default_channel_layout (c->output_channels);
+
+    ret = av_opt_set_bin (c->sink,
+                          "channel_layouts",
+                          (uint8_t *) &channel_layout,
+                          sizeof (channel_layout),
+                          AV_OPT_SEARCH_CHILDREN);
+    if (ret < 0)
+      {
+        edc_rtsp_debug (rtsp, "%s", "failed to set output channel layout");
+        return -1;
+      }
+  }
+
+  ret = av_opt_set_bin (c->sink,
+                        "sample_rates",
+                        (uint8_t *) &c->output_sample_rate,
+                        sizeof (c->output_sample_rate),
+                        AV_OPT_SEARCH_CHILDREN);
+  if (ret < 0)
+    {
+      edc_rtsp_debug (rtsp, "%s", "failed to set output sample rate");
+      return -1;
+    }
+
+  outputs = avfilter_inout_alloc ();
+  outputs->name = av_strdup ("in");
+  outputs->filter_ctx = c->src;
+  outputs->pad_idx = 0;
+  outputs->next = NULL;
+
+  inputs = avfilter_inout_alloc ();
+  inputs->name = av_strdup ("out");
+  inputs->filter_ctx = c->sink;
+  inputs->pad_idx = 0;
+  inputs->next = NULL;
+
+  do
+    {
+      ret = avfilter_graph_parse_ptr (c->graph,
+                                      "aresample",
+                                      &inputs,
+                                      &outputs,
+                                      NULL);
+      if (ret < 0)
+        {
+          edc_rtsp_debug (rtsp, "failed to parse audio filter graph : '%s'(%d)",
+                         av_err2str (ret), ret);
+          avfilter_graph_free (&c->graph);
+          break;
+        }
+
+      ret = avfilter_graph_config (c->graph, NULL);
+      if (ret < 0)
+        {
+          edc_rtsp_debug (rtsp, "failed to configure audio filter graph : '%s'(%d)",
+                         av_err2str (ret), ret);
+          avfilter_graph_free (&c->graph);
+          break;
+        }
+
+      if (!(c->encoder->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE))
+        av_buffersink_set_frame_size (c->sink, c->encoder->frame_size);
+
+      if (1)
+        {
+          char *dump;
+
+          dump = avfilter_graph_dump (c->graph, NULL);
+          if (dump)
+            {
+              edc_rtsp_debug (rtsp, "resampling filter graph\n%s", dump);
+              av_free (dump);
+            }
+        }
+
+      ret = 0;
+    }
+  while (0);
+
+  avfilter_inout_free (&inputs);
+  avfilter_inout_free (&outputs);
+
+  return ret;
+}
+
 static void
 edc_rtsp_handle_error (EdcRtsp    *rtsp,
                        const char *fmt,
@@ -567,6 +746,9 @@ edc_rtsp_read_rtp_real (EdcRtsp  *rtsp,
   if (!pkt || pkt->size <= 0 || !pkt->data)
     return NULL;
 
+  if (rtsp->base_timestamp == 0)
+    rtsp->base_timestamp = edc_get_real_time ();
+
   /* RTP extension header */
   if (rtsp->fmt.avfctx->rtp_ext_header_len > 0)
     {
@@ -584,19 +766,37 @@ edc_rtsp_read_rtp_real (EdcRtsp  *rtsp,
   buf = NULL;
   if (pkt->stream_index == rtsp->fmt.video_stream)
     {
+      AVRational tb = { 1, 90000 };
+
+      if (rtsp->video.start_pts < 0)
+        rtsp->video.start_pts = pkt->pts;
+
       if (!rtsp->video.has_stream ||
           !rtsp->use_stream[EDC_RTSP_VIDEO])
         return NULL;
 
+      if (rtsp->video.start_pts > pkt->pts)
+        rtsp->video.start_pts = pkt->pts;
+
+      pkt->pts -= rtsp->video.start_pts;
+
+      if (0) edc_rtsp_log (rtsp, "video time:%s pts:%ld", av_ts2timestr (pkt->pts, &tb), pkt->pts);
+
       buf = rtsp->video.make_frame (rtsp, pkt);
       if (buf)
         {
           buf->width = rtsp->video.width;
           buf->height = rtsp->video.height;
+          buf->timestamp = rtsp->base_timestamp + av_rescale_q (pkt->pts, tb, AV_TIME_BASE_Q);
         }
     }
   else if (pkt->stream_index == rtsp->fmt.audio_stream)
     {
+      AVRational tb = { 1, 44100 };
+
+      if (rtsp->audio.start_pts < 0)
+        rtsp->audio.start_pts = pkt->pts;
+
       if (!rtsp->audio.has_stream ||
           !rtsp->use_stream[EDC_RTSP_AUDIO])
         return NULL;
@@ -604,7 +804,13 @@ edc_rtsp_read_rtp_real (EdcRtsp  *rtsp,
       if (rtsp->ignore_audio_frame)
         return NULL;
 
+      pkt->pts -= rtsp->audio.start_pts;
+
+      if (0) edc_rtsp_log (rtsp, "audio time:%s pts:%ld", av_ts2timestr (pkt->pts, &tb), pkt->pts);
+
       buf = rtsp->audio.make_frame (rtsp, pkt);
+      if (buf)
+        buf->timestamp = rtsp->base_timestamp + av_rescale_q (pkt->pts, tb, AV_TIME_BASE_Q);
     }
   else if (pkt->stream_index == rtsp->fmt.metadata_stream)
     {
@@ -912,6 +1118,7 @@ static EdcBuf *
 edc_rtsp_make_audio_frame_resampled (EdcRtsp  *rtsp,
                                      AVPacket *pkt)
 {
+#if 0
   EdcRtspConvert *c;
   EdcBuf *buf;
   int len, pcm_samples, pcm_len;
@@ -970,6 +1177,173 @@ edc_rtsp_make_audio_frame_resampled (EdcRtsp  *rtsp,
   buf->codec_id = EDC_CODEC_ADPCM;
   buf->len = len;
 
+  return buf;
+#else
+  return NULL;
+#endif
+}
+
+static EdcBuf *
+edc_rtsp_make_audio_frame_resampled2 (EdcRtsp  *rtsp,
+                                      AVPacket *pkt)
+{
+  EdcRtspConvert *c;
+  EdcBuf *buf = NULL;
+  int pcm_len;
+  const uint8_t **dec_data;
+  int dec_data_nb_samples;
+  int ret;
+
+  c = &rtsp->audio.convert;
+
+  dec_data = edc_codec_decode_audio2 (c->decoder,
+                                      (char *) pkt->data,
+                                      (int) pkt->size,
+                                      &dec_data_nb_samples);
+  if (!dec_data)
+    {
+      int error_count;
+
+      error_count = edc_codec_get_error_count (c->decoder);
+      if (error_count <= 5 || (error_count % 100) == 0)
+        edc_rtsp_log (rtsp,
+                      "failed to decode audio frame(%d) (count:%d)",
+                      pkt->size, error_count);
+
+      edc_rtsp_handle_error (rtsp, "Audio Resample");
+
+      return NULL;
+    }
+
+  {
+    AVRational tb = (AVRational) { 1, c->output_sample_rate };
+    AVFrame *frame;
+
+    frame = av_frame_alloc ();
+    frame->pts = pkt->pts;
+    frame->pkt_dts = frame->pts;
+    frame->channel_layout = av_get_default_channel_layout (c->input_channels);
+    frame->channels = av_get_channel_layout_nb_channels (frame->channel_layout);
+    frame->sample_rate = c->input_sample_rate;
+    frame->format = c->input_sample_fmt;
+    frame->nb_samples = dec_data_nb_samples;
+
+    avcodec_fill_audio_frame (frame,
+                              frame->channels,
+                              frame->format,
+                              (const uint8_t *) dec_data[0],
+                              192000,
+                              0);
+
+    if (0) edc_log ("audio decoded tb(%d/%d) nb_samples:%d sample_rate:%d fmt:%d layout:%ld time:%s pts:%s pkt_dts:%s",
+                    tb.num,
+                    tb.den,
+                    frame->nb_samples,
+                    frame->sample_rate,
+                    frame->format,
+                    frame->channel_layout,
+                    av_ts2timestr (frame->pts, &tb),
+                    av_ts2str (frame->pts),
+                    av_ts2str (frame->pkt_dts));
+
+    /* send frame to the audio resampler */
+    ret = av_buffersrc_add_frame_flags (c->src,
+                                        frame,
+                                        AV_BUFFERSRC_FLAG_KEEP_REF);
+    if (ret < 0)
+      edc_rtsp_debug (rtsp, "failed to add frame to buffersrc: '%s'(%d)",
+                      av_err2str (ret), ret);
+
+    av_frame_free (&frame);
+  }
+
+  while (TRUE)
+    {
+      AVFrame *frame;
+
+      frame = av_frame_alloc ();
+
+      ret = av_buffersink_get_frame (c->sink, frame);
+      if (ret < 0)
+        {
+          if (ret != AVERROR (EAGAIN) && ret != AVERROR_EOF)
+            edc_rtsp_debug (rtsp, "failed to get frame from buffersink: '%s'(%d)",
+                            av_err2str (ret), ret);
+
+          av_frame_free (&frame);
+          break;
+        }
+
+      if (0)
+        {
+          AVRational tb;
+
+          tb = av_buffersink_get_time_base (c->sink);
+          edc_log ("audio resampled tb(%d/%d) nb_samples:%d sample_rate:%d fmt:%d layout:%ld time:%s pts:%s pkt_dts:%s",
+                  tb.num, tb.den,
+                  frame->nb_samples,
+                  frame->sample_rate,
+                  frame->format,
+                  frame->channel_layout,
+                  av_ts2timestr (frame->pts, &tb),
+                  av_ts2str (frame->pts),
+                  av_ts2str (frame->pkt_dts));
+        }
+
+      pcm_len = frame->nb_samples * c->output_bytes_per_sample * c->output_channels;
+      if (pcm_len <= 0)
+        {
+          av_frame_free (&frame);
+          break;
+        }
+
+      ret = avcodec_send_frame (c->encoder, frame);
+      if (ret < 0)
+        {
+          if (ret != AVERROR_EOF)
+            edc_rtsp_log (rtsp, "failed to encode audio frame: '%s'(%d)",
+                          av_err2str (ret), ret);
+        }
+
+      while (TRUE)
+        {
+          AVPacket enc_pkt;
+
+          av_init_packet (&enc_pkt);
+
+          ret = avcodec_receive_packet (c->encoder, &enc_pkt);
+          if (ret < 0)
+            {
+              if (ret != AVERROR (EAGAIN) && ret != AVERROR_EOF)
+                edc_rtsp_log (rtsp, "failed to get encoded audio frame: '%s'(%d)",
+                              av_err2str (ret), ret);
+
+              av_packet_unref (&enc_pkt);
+              break;
+            }
+
+          if (enc_pkt.size <= 0)
+            {
+              edc_rtsp_log (rtsp, "failed to get encoded frame");
+              av_packet_unref (&enc_pkt);
+              break;
+            }
+
+          buf = edc_buf_new (enc_pkt.size + AV_INPUT_BUFFER_PADDING_SIZE);
+          memcpy (buf->data, enc_pkt.data, enc_pkt.size);
+          buf->len = enc_pkt.size;
+          buf->channel = 0;
+          buf->flags = EDC_BUF_FLAG_AUDIO;
+          buf->codec_id = EDC_CODEC_ADPCM;
+
+          av_packet_unref (&enc_pkt);
+          break;
+        }
+
+      av_frame_free (&frame);
+      break;
+    }
+
   return buf;
 }
 
@@ -977,6 +1351,7 @@ static EdcBuf *
 edc_rtsp_make_audio_frame_encoded (EdcRtsp  *rtsp,
                                    AVPacket *pkt)
 {
+#if 0
   EdcRtspConvert *c;
   EdcBuf *buf;
   int ret, pcm_len, len;
@@ -1021,6 +1396,9 @@ edc_rtsp_make_audio_frame_encoded (EdcRtsp  *rtsp,
   buf->len = len;
 
   return buf;
+#else
+  return NULL;
+#endif
 }
 
 static EdcBuf *
@@ -1483,16 +1861,57 @@ edc_rtsp_open_fmt (EdcRtsp *rtsp)
           edc_codec_set_sample_rate (c->decoder, c->input_sample_rate);
           edc_codec_set_channels (c->decoder, c->input_channels);
 
-          c->encoder = edc_adpcm_new ();
-          if (!c->encoder)
-            {
-              rtsp->audio.has_stream = 0;
-
-              edc_rtsp_log (rtsp,
-                            "failed to create encoder '%d'",
-                            c->output_codec_id);
-              continue;
-            }
+          {
+            AVCodec *enc;
+            AVCodecParameters *par;
+
+            enc = avcodec_find_encoder (AV_CODEC_ID_ADPCM_IMA_WAV);
+            if (!enc)
+              {
+                rtsp->audio.has_stream = 0;
+                edc_rtsp_log (rtsp, "failed to find ADPCM encoder...");
+                continue;
+              }
+
+            c->encoder = avcodec_alloc_context3 (enc);
+            if (!c->encoder)
+              {
+                rtsp->audio.has_stream = 0;
+
+                edc_rtsp_log (rtsp, "failed to create encoder '%d'", c->output_codec_id);
+                continue;
+              }
+
+            par = avcodec_parameters_alloc ();
+            par->codec_type = AVMEDIA_TYPE_AUDIO;
+            par->codec_id = enc->id;
+            par->codec_tag = 17;
+            par->format = AV_SAMPLE_FMT_S16;
+            par->bit_rate = 32000;
+            par->bits_per_coded_sample = 4;
+            par->bits_per_raw_sample = 0;
+            par->sample_rate = 8000;
+            par->channel_layout = AV_CH_LAYOUT_MONO;
+            par->channels =
+              av_get_channel_layout_nb_channels (par->channel_layout);
+            par->block_align = 1024;
+
+            avcodec_parameters_to_context (c->encoder, par);
+            avcodec_parameters_free (&par);
+
+            ret = avcodec_open2 (c->encoder, enc, NULL);
+            if (ret < 0)
+              {
+                rtsp->audio.has_stream = 0;
+                edc_rtsp_log (rtsp, "failed to open audio encoder: '%s'(%d)",
+                              av_err2str (ret), ret);
+
+                if (c->encoder)
+                  avcodec_free_context (&c->encoder);
+
+                continue;
+              }
+          }
 
           /* memory for store decoded/resampledconverted data */
           c->d.alloc_size = EDC_CODEC_MAX_AUDIO_FRAME_SIZE;
@@ -1526,7 +1945,13 @@ edc_rtsp_open_fmt (EdcRtsp *rtsp)
                   continue;
                 }
 
-              rtsp->audio.make_frame = edc_rtsp_make_audio_frame_resampled;
+              if (0)
+                rtsp->audio.make_frame = edc_rtsp_make_audio_frame_resampled;
+              else
+                {
+                  mp4serv_init_audio_resampling_filter (rtsp);
+                  rtsp->audio.make_frame = edc_rtsp_make_audio_frame_resampled2;
+                }
             }
           else
             {
@@ -1642,7 +2067,7 @@ edc_rtsp_close_fmt (EdcRtsp *rtsp)
 
   if (c->encoder)
     {
-      edc_adpcm_unref (c->encoder);
+      avcodec_free_context (&c->encoder);
       c->encoder = NULL;
     }
 
@@ -1891,6 +2316,9 @@ edc_rtsp_start (EdcRtsp *rtsp)
     }
   rtsp->started = TRUE;
   rtsp->key_received = FALSE;
+  rtsp->base_timestamp = 0;
+  rtsp->video.start_pts = -1;
+  rtsp->audio.start_pts = -1;
 
   edc_reset_rel_timestamp (&rtsp->rel_timestamp);