ao_avf.diff

From 5c01f4d2c6f097275d0f5b650d93138ce51b25d9 Mon Sep 17 00:00:00 2001
From: Aman Gupta <aman@tmm1.net>
Date: Wed, 23 Jan 2019 16:30:57 -0800
Subject: [PATCH 1/3] audio/out: add driver for AVFoundation's
 AVSampleBufferAudioRenderer

---
 audio/out/ao.c              |   4 +
 audio/out/ao_avfoundation.m | 254 ++++++++++++++++++++++++++++++++++++
 wscript                     |   5 +
 wscript_build.py            |   5 +-
 4 files changed, 266 insertions(+), 2 deletions(-)
 create mode 100644 audio/out/ao_avfoundation.m

diff --git a/audio/out/ao.c b/audio/out/ao.c
index 86488c20ef..5fc981409b 100644
--- a/audio/out/ao.c
+++ b/audio/out/ao.c
@@ -38,6 +38,7 @@
 extern const struct ao_driver audio_out_oss;
 extern const struct ao_driver audio_out_audiotrack;
 extern const struct ao_driver audio_out_audiounit;
+extern const struct ao_driver audio_out_avfoundation;
 extern const struct ao_driver audio_out_coreaudio;
 extern const struct ao_driver audio_out_coreaudio_exclusive;
 extern const struct ao_driver audio_out_rsound;
@@ -64,6 +65,9 @@ static const struct ao_driver * const audio_out_drivers[] = {
 #if HAVE_COREAUDIO
     &audio_out_coreaudio,
 #endif
+#if HAVE_AVFOUNDATION
+    &audio_out_avfoundation,
+#endif
 #if HAVE_PULSE
     &audio_out_pulse,
 #endif
diff --git a/audio/out/ao_avfoundation.m b/audio/out/ao_avfoundation.m
new file mode 100644
index 0000000000..ad1b2ce32b
--- /dev/null
+++ b/audio/out/ao_avfoundation.m
@@ -0,0 +1,254 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "config.h"
+#include "ao.h"
+#include "internal.h"
+#include "audio/format.h"
+#include "osdep/timer.h"
+#include "options/m_option.h"
+#include "misc/ring.h"
+#include "common/msg.h"
+#include "ao_coreaudio_utils.h"
+#include "ao_coreaudio_chmap.h"
+
+#import <CoreAudio/CoreAudioTypes.h>
+#import <AudioToolbox/AudioToolbox.h>
+#import <AVFoundation/AVFoundation.h>
+#import <mach/mach_time.h>
+
+#if TARGET_OS_IPHONE
+#define HAVE_AVAUDIOSESSION
+#endif
+
+struct priv {
+    dispatch_queue_t queue;
+    CMFormatDescriptionRef desc;
+    AVSampleBufferAudioRenderer *renderer;
+    AVSampleBufferRenderSynchronizer *synchronizer;
+    int64_t enqueued;
+};
+
+static bool enqueue_frames(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+    int frames = ao->buffer / 4;
+    int size = frames * ao->sstride;
+    OSStatus err;
+
+    CMSampleBufferRef sBuf = NULL;
+    CMBlockBufferRef bBuf = NULL;
+
+    int64_t playhead = CMTimeGetSeconds([p->synchronizer currentTime]) * 1e6;
+    int64_t end = mp_time_us();
+    end += ca_frames_to_us(ao, frames);
+    end += MPMAX(0, ca_frames_to_us(ao, p->enqueued) - playhead);
+    void *buf = calloc(size, 1);
+    int samples = ao_read_data(ao, &buf, frames, end);
+
+    if (samples <= 0) {
+        free(buf);
+        return false;
+    }
+
+    int bufsize = samples * ao->sstride;
+    err = CMBlockBufferCreateWithMemoryBlock(NULL,    // structureAllocator
+                                             buf,     // memoryBlock
+                                             bufsize, // blockLength
+                                             0,       // blockAllocator
+                                             NULL,    // customBlockSource
+                                             0,       // offsetToData
+                                             bufsize, // dataLength
+                                             0,       // flags
+                                             &bBuf);
+    CHECK_CA_WARN("failed to create CMBlockBuffer");
+
+    p->enqueued += samples;
+    CMSampleTimingInfo timing = {
+        CMTimeMake(1, ao->samplerate),
+        CMTimeMake(p->enqueued, ao->samplerate),
+        kCMTimeInvalid
+    };
+    err = CMSampleBufferCreate(NULL,    // allocator
+                               bBuf,    // dataBuffer
+                               true,    // dataReady
+                               NULL,    // makeDataReadyCallback
+                               NULL,    // makeDataReadyRefcon
+                               p->desc, // formatDescription
+                               bufsize, // numSamples
+                               1,       // numSampleTimingEntries
+                               &timing, // sampleTimingArray
+                               0,       // numSampleSizeEntries
+                               NULL,    // sampleSizeArray
+                               &sBuf);
+    CHECK_CA_WARN("failed to create CMSampleBuffer");
+
+    [p->renderer enqueueSampleBuffer:sBuf];
+
+    CFRelease(sBuf);
+    CFRelease(bBuf);
+    return true;
+}
+
+static void play(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+
+    [p->renderer requestMediaDataWhenReadyOnQueue:p->queue usingBlock:^{
+        while ([p->renderer isReadyForMoreMediaData]) {
+            if (!enqueue_frames(ao))
+                break;
+        }
+    }];
+}
+
+static bool init_renderer(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+    OSStatus err;
+    AudioStreamBasicDescription asbd;
+
+#ifdef HAVE_AVAUDIOSESSION
+    AVAudioSession *instance = AVAudioSession.sharedInstance;
+    AVAudioSessionPortDescription *port = nil;
+    NSInteger maxChannels = instance.maximumOutputNumberOfChannels;
+    NSInteger prefChannels = MIN(maxChannels, ao->channels.num);
+
+    [instance
+        setCategory:AVAudioSessionCategoryPlayback
+        mode:AVAudioSessionModeMoviePlayback
+        routeSharingPolicy:AVAudioSessionRouteSharingPolicyLongForm
+        options:0
+        error:nil];
+    [instance setActive:YES error:nil];
+    [instance setPreferredOutputNumberOfChannels:prefChannels error:nil];
+
+    if (af_fmt_is_spdif(ao->format) || instance.outputNumberOfChannels <= 2) {
+        ao->channels = (struct mp_chmap)MP_CHMAP_INIT_STEREO;
+    } else {
+        port = instance.currentRoute.outputs.firstObject;
+        if (port.channels.count == 2 &&
+            port.portType == AVAudioSessionPortHDMI) {
+            // Special case when using an HDMI adapter. The iOS device will
+            // perform SPDIF conversion for us, so send all available channels
+            // using the AC3 mapping.
+            ao->channels = (struct mp_chmap)MP_CHMAP6(FL, FC, FR, SL, SR, LFE);
+        } else {
+            ao->channels.num = (uint8_t)port.channels.count;
+            for (AVAudioSessionChannelDescription *ch in port.channels) {
+              ao->channels.speaker[ch.channelNumber - 1] =
+                  ca_label_to_mp_speaker_id(ch.channelLabel);
+            }
+        }
+    }
+#else
+    // todo: support multi-channel on macOS
+    ao->channels = (struct mp_chmap)MP_CHMAP_INIT_STEREO;
+#endif
+
+    // todo: add support for planar formats to play()
+    ao->format = af_fmt_from_planar(ao->format);
+    ca_fill_asbd(ao, &asbd);
+    err = CMAudioFormatDescriptionCreate(NULL,
+                                         &asbd,
+                                         0, NULL,
+                                         0, NULL,
+                                         NULL,
+                                         &p->desc);
+    CHECK_CA_ERROR_L(coreaudio_error,
+                     "unable to create format description");
+
+    return true;
+
+coreaudio_error:
+    return false;
+}
+
+static void stop(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+    dispatch_sync(p->queue, ^{
+        [p->synchronizer setRate:0.0 time:CMTimeMake(0, ao->samplerate)];
+        [p->renderer stopRequestingMediaData];
+        [p->renderer flush];
+        p->enqueued = 0;
+    });
+}
+
+static void start(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+    dispatch_async(p->queue, ^{
+        int n = 0;
+        while (enqueue_frames(ao)) n++;
+        play(ao);
+        [p->synchronizer setRate:1.0];
+    });
+}
+
+static void uninit(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+
+    p->renderer = nil;
+    p->synchronizer = nil;
+    if (p->desc) {
+        CFRelease(p->desc);
+        p->desc = NULL;
+    }
+
+#ifdef HAVE_AVAUDIOSESSION
+    [AVAudioSession.sharedInstance
+        setActive:NO
+        withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation
+        error:nil];
+#endif
+}
+
+static int init(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+
+    if (@available(tvOS 12.0, iOS 12.0, macOS 10.14, *)) {
+        // supported, fall through
+    } else {
+        MP_FATAL(ao, "unsupported on this OS version\n");
+        return CONTROL_ERROR;
+    }
+
+    p->queue = dispatch_queue_create("mpv audio renderer", NULL);
+    p->renderer = [AVSampleBufferAudioRenderer new];
+    p->synchronizer = [AVSampleBufferRenderSynchronizer new];
+    [p->synchronizer addRenderer:p->renderer];
+
+    if (!init_renderer(ao))
+        return CONTROL_ERROR;
+
+    return CONTROL_OK;
+}
+
+#define OPT_BASE_STRUCT struct priv
+
+const struct ao_driver audio_out_avfoundation = {
+    .description    = "AVFoundation AVSampleBufferAudioRenderer (macOS/iOS)",
+    .name           = "avfoundation",
+    .uninit         = uninit,
+    .init           = init,
+    .reset          = stop,
+    .resume         = start,
+    .priv_size      = sizeof(struct priv),
+};
diff --git a/wscript b/wscript
index 0b58934fe2..e083adefb0 100644
--- a/wscript
+++ b/wscript
@@ -516,6 +516,11 @@ audio_output_features = [
         'name': '--alsa',
         'desc': 'ALSA audio output',
         'func': check_pkg_config('alsa', '>= 1.0.18'),
+    }, {
+        'name': '--avfoundation',
+        'desc': 'AVFoundation audio output',
+        'deps': 'atomics',
+        'func': check_cc(framework_name=['AVFoundation', 'CoreMedia', 'AudioToolbox']),
     }, {
         'name': '--coreaudio',
         'desc': 'CoreAudio audio output',
diff --git a/wscript_build.py b/wscript_build.py
index 711c293f5b..fa702f9d3f 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -228,11 +228,12 @@ def build(ctx):
         ( "audio/out/ao_alsa.c",                 "alsa" ),
         ( "audio/out/ao_audiotrack.c",           "android" ),
         ( "audio/out/ao_audiounit.m",            "audiounit" ),
+        ( "audio/out/ao_avfoundation.m",         "avfoundation" ),
         ( "audio/out/ao_coreaudio.c",            "coreaudio" ),
-        ( "audio/out/ao_coreaudio_chmap.c",      "coreaudio || audiounit" ),
+        ( "audio/out/ao_coreaudio_chmap.c",      "coreaudio || audiounit || avfoundation" ),
         ( "audio/out/ao_coreaudio_exclusive.c",  "coreaudio" ),
         ( "audio/out/ao_coreaudio_properties.c", "coreaudio" ),
-        ( "audio/out/ao_coreaudio_utils.c",      "coreaudio || audiounit" ),
+        ( "audio/out/ao_coreaudio_utils.c",      "coreaudio || audiounit || avfoundation" ),
         ( "audio/out/ao_jack.c",                 "jack" ),
         ( "audio/out/ao_lavc.c" ),
         ( "audio/out/ao_null.c" ),
-- 
2.20.1


From 20fcb7e2b199c1d56854b2e9036d200a6ffce2b5 Mon Sep 17 00:00:00 2001
From: Aman Gupta <aman@tmm1.net>
Date: Fri, 1 Feb 2019 16:57:43 -0800
Subject: [PATCH 2/3] ao/pull: add optional get_delay and pause callbacks

useful for underlying audio drivers with dynamic buffers,
which don't necessarily read the audio source in real time.

when a large buffer is built up, the pause callback helps optimize
the pause/resume case by avoiding a buffer flush.
---
 audio/out/internal.h |  2 ++
 audio/out/pull.c     | 17 ++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/audio/out/internal.h b/audio/out/internal.h
index 7eba7f26cf..5363bb69f0 100644
--- a/audio/out/internal.h
+++ b/audio/out/internal.h
@@ -148,6 +148,7 @@ struct ao_driver {
     // pull based: stop the audio callback
     void (*reset)(struct ao *ao);
     // push based: see ao_pause()
+    // pull based: optional
     void (*pause)(struct ao *ao);
     // push based: see ao_resume()
     // pull based: start the audio callback
@@ -157,6 +158,7 @@ struct ao_driver {
     // push based: see ao_play()
     int (*play)(struct ao *ao, void **data, int samples, int flags);
     // push based: see ao_get_delay()
+    // pull based: optional
     double (*get_delay)(struct ao *ao);
     // push based: block until all queued audio is played (optional)
     void (*drain)(struct ao *ao);
diff --git a/audio/out/pull.c b/audio/out/pull.c
index 6af087259d..2756481430 100644
--- a/audio/out/pull.c
+++ b/audio/out/pull.c
@@ -239,10 +239,15 @@ static int control(struct ao *ao, enum aocontrol cmd, void *arg)
 static double get_delay(struct ao *ao)
 {
     struct ao_pull_state *p = ao->api_priv;
-
-    int64_t end = atomic_load(&p->end_time_us);
-    int64_t now = mp_time_us();
-    double driver_delay = MPMAX(0, (end - now) / (1000.0 * 1000.0));
+    double driver_delay = 0.0;
+
+    if (ao->driver->get_delay) {
+        driver_delay = ao->driver->get_delay(ao);
+    } else {
+        int64_t end = atomic_load(&p->end_time_us);
+        int64_t now = mp_time_us();
+        driver_delay = MPMAX(0, (end - now) / (1000.0 * 1000.0));
+    }
     return mp_ring_buffered(p->buffers[0]) / (double)ao->bps + driver_delay;
 }
 
@@ -259,7 +264,9 @@ static void reset(struct ao *ao)
 
 static void pause(struct ao *ao)
 {
-    if (!ao->stream_silence && ao->driver->reset)
+    if (!ao->stream_silence && ao->driver->pause)
+        ao->driver->pause(ao);
+    else if (!ao->stream_silence && ao->driver->reset)
         ao->driver->reset(ao);
     set_state(ao, AO_STATE_NONE);
 }
-- 
2.20.1


From 37ca2420f859819965a77f0bef7a932cf9dd938c Mon Sep 17 00:00:00 2001
From: Aman Gupta <aman@tmm1.net>
Date: Fri, 1 Feb 2019 17:00:43 -0800
Subject: [PATCH 3/3] ao/avfoundation: use new delay/pause callbacks

---
 audio/out/ao_avfoundation.m | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/audio/out/ao_avfoundation.m b/audio/out/ao_avfoundation.m
index ad1b2ce32b..eb54018d26 100644
--- a/audio/out/ao_avfoundation.m
+++ b/audio/out/ao_avfoundation.m
@@ -53,13 +53,8 @@ static bool enqueue_frames(struct ao *ao)
     CMSampleBufferRef sBuf = NULL;
     CMBlockBufferRef bBuf = NULL;
 
-    int64_t playhead = CMTimeGetSeconds([p->synchronizer currentTime]) * 1e6;
-    int64_t end = mp_time_us();
-    end += ca_frames_to_us(ao, frames);
-    end += MPMAX(0, ca_frames_to_us(ao, p->enqueued) - playhead);
     void *buf = calloc(size, 1);
-    int samples = ao_read_data(ao, &buf, frames, end);
-
+    int samples = ao_read_data(ao, &buf, frames, 0 /*unused*/);
     if (samples <= 0) {
         free(buf);
         return false;
@@ -178,6 +173,15 @@ coreaudio_error:
     return false;
 }
 
+static void pause_no_flush(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+    dispatch_sync(p->queue, ^{
+        [p->synchronizer setRate:0.0];
+        [p->renderer stopRequestingMediaData];
+    });
+}
+
 static void stop(struct ao *ao)
 {
     struct priv *p = ao->priv;
@@ -241,6 +245,15 @@ static int init(struct ao *ao)
     return CONTROL_OK;
 }
 
+static double get_delay(struct ao *ao)
+{
+    struct priv *p = ao->priv;
+
+    int64_t playhead = CMTimeGetSeconds([p->synchronizer currentTime]) * 1e6;
+    double delay = (ca_frames_to_us(ao, p->enqueued) - playhead) / (double)(1e6);
+    return delay;
+}
+
 #define OPT_BASE_STRUCT struct priv
 
 const struct ao_driver audio_out_avfoundation = {
@@ -250,5 +263,7 @@ const struct ao_driver audio_out_avfoundation = {
     .init           = init,
     .reset          = stop,
     .resume         = start,
+    .pause          = pause_no_flush,
+    .get_delay      = get_delay,
     .priv_size      = sizeof(struct priv),
 };
-- 
2.20.1