Note for coreaudio

README.md

CoreAudio issue

This post is to record issues I found on CoreAudio service. The AudioStream is a light audio backend for demo. The test_* are the known issues.

• test_audio: Verify the sanity of AudioStream
• test_deadlock: Reproduce the known deadlock issue

Deadlock

How to use

Clone this repo and run $ make all. You can use $ make clean

TODO

• Implement a state callback to notify started, stopped, or drained
  • Maybe we should turn AudioStream into FSM style

AudioStream.cpp

#include <assert.h>
#include <CoreAudio/CoreAudio.h>

#include "AudioStream.h"

#define AU_OUT_BUS  0
// #define AU_IN_BUS   1

AudioStream::AudioStream(Format aFormat,
                         unsigned int aRate,
                         unsigned int aChannels,
                         AudioCallback aCallback)
  : mRate(aRate)
  , mChannels(aChannels)
  , mUnit(nullptr)
  , mCallback(aCallback)
{
  assert(mRate && mChannels);
  CreateAudioUnit(); // Initialize mUnit
  assert(SetDescription(aFormat)); // Initialize mDescription
  assert(SetCallback()); // Render output to DataCallback
  assert(AudioUnitInitialize(mUnit) == noErr);
}

AudioStream::~AudioStream()
{
  assert(mUnit);
  assert(AudioOutputUnitStop(mUnit) == noErr);
  assert(AudioUnitUninitialize(mUnit) == noErr);
  assert(AudioComponentInstanceDispose(mUnit) == noErr);
}

void
AudioStream::Start()
{
  assert(mUnit);
  assert(AudioOutputUnitStart(mUnit) == noErr);
}

void
AudioStream::Stop()
{
  assert(mUnit);
  assert(AudioOutputUnitStop(mUnit) == noErr);
}

void
AudioStream::CreateAudioUnit()
{
  assert(!mUnit); // mUnit should be nullptr before initializing.

  AudioComponentDescription desc;
  desc.componentType = kAudioUnitType_Output;
  desc.componentSubType = kAudioUnitSubType_DefaultOutput;
  desc.componentManufacturer = kAudioUnitManufacturer_Apple;
  desc.componentFlags = 0;
  desc.componentFlagsMask = 0;

  AudioComponent comp = AudioComponentFindNext(NULL, &desc);
  assert(comp); // comp will be nullptr if there is no matching audio hardware.

  assert(AudioComponentInstanceNew(comp, &mUnit) == noErr);
  assert(mUnit); // mUnit should NOT be nullptr after initializing.
}

bool
AudioStream::SetDescription(Format aFormat)
{
  memset(&mDescription, 0, sizeof(mDescription));
  switch (aFormat) {
    case S16LE:
      mDescription.mBitsPerChannel = 16;
      mDescription.mFormatFlags = kAudioFormatFlagIsSignedInteger;
      break;
    case S16BE:
      mDescription.mBitsPerChannel = 16;
      mDescription.mFormatFlags = kAudioFormatFlagIsSignedInteger |
                                  kAudioFormatFlagIsBigEndian;
      break;
    case F32LE:
      mDescription.mBitsPerChannel = 32;
      mDescription.mFormatFlags = kAudioFormatFlagIsFloat;
      break;
    case F32BE:
      mDescription.mBitsPerChannel = 32;
      mDescription.mFormatFlags = kAudioFormatFlagIsFloat |
                                  kAudioFormatFlagIsBigEndian;
      break;
    default:
      return false;
  }

  // The mFormatFlags below should be set by "|" or operator,
  // or the assigned flags above will be cleared.
  mDescription.mFormatID = kAudioFormatLinearPCM;
  mDescription.mFormatFlags |= kLinearPCMFormatFlagIsPacked;
  mDescription.mSampleRate = mRate;
  mDescription.mChannelsPerFrame = mChannels;

  mDescription.mBytesPerFrame = (mDescription.mBitsPerChannel / 8) *
                                mDescription.mChannelsPerFrame;

  mDescription.mFramesPerPacket = 1;
  mDescription.mBytesPerPacket = mDescription.mBytesPerFrame *
                                 mDescription.mFramesPerPacket;
  mDescription.mReserved = 0;

  return AudioUnitSetProperty(mUnit,
                              kAudioUnitProperty_StreamFormat,
                              kAudioUnitScope_Input,
                              AU_OUT_BUS,
                              &mDescription,
                              sizeof(mDescription)) == noErr;
}

bool
AudioStream::SetCallback()
{
  AURenderCallbackStruct aurcbs;
  memset(&aurcbs, 0, sizeof(aurcbs));
  aurcbs.inputProc = DataCallback;
  aurcbs.inputProcRefCon = this; // Pass this as callback's arguments

  return AudioUnitSetProperty(mUnit,
                              kAudioUnitProperty_SetRenderCallback,
                              kAudioUnitScope_Global,
                              AU_OUT_BUS,
                              &aurcbs,
                              sizeof(aurcbs)) == noErr;
}

/* static */ OSStatus
AudioStream::DataCallback(void* aRefCon,
                          AudioUnitRenderActionFlags* aActionFlags,
                          const AudioTimeStamp* aTimeStamp,
                          UInt32 aBusNumber,
                          UInt32 aNumFrames,
                          AudioBufferList* aData)
{
  assert(aBusNumber == AU_OUT_BUS);
  assert(aData->mNumberBuffers == 1);

  AudioStream* as = static_cast<AudioStream*>(aRefCon); // Get arguments
  void* buffer = aData->mBuffers[0].mData;
  as->mCallback(buffer, aNumFrames);
  return noErr;
}

AudioStream.h

#ifndef AUDIOSTREAM_H
#define AUDIOSTREAM_H

#include <AudioUnit/AudioUnit.h>

typedef void (* AudioCallback)(void* buffer, unsigned long frames);

class AudioStream
{
public:
  // We only support output for now.
  // enum Side
  // {
  //   OUTPUT,
  //   INPUT
  // }

  enum Format
  {
    S16LE, // PCM signed 16-bit little-endian
    S16BE, // PCM signed 16-bit big-endian
    F32LE, // PCM 32-bit floating-point little-endian
    F32BE  // PCM 32-bit floating-point big-endian
  };

  AudioStream(Format aFormat,
              unsigned int aRate,
              unsigned int aChannels,
              AudioCallback aCallback);

  ~AudioStream();

  void Start();
  void Stop();

private:
  void CreateAudioUnit();
  bool SetDescription(Format aFormat);
  bool SetCallback();
  static OSStatus DataCallback(void* aRefCon,
                               AudioUnitRenderActionFlags* aActionFlags,
                               const AudioTimeStamp* aTimeStamp,
                               UInt32 aBusNumber,
                               UInt32 aNumFrames,
                               AudioBufferList* aData);

  unsigned int mRate;
  unsigned int mChannels;
  AudioStreamBasicDescription mDescription; // Format descriptions
  // AudioUnit is a pointer to ComponentInstanceRecord
  AudioUnit mUnit;
  AudioCallback mCallback;
};

#endif // #ifndef AUDIOSTREAM_H

deadlock-1.png

deadlock-2.png

deadlock-3.png

deadlock-4.png

deadlock.gif

makefile

CXX=g++
CFLAGS=-Wall -std=c++14
LIBRARIES=-framework CoreAudio -framework AudioUnit

SOURCES=AudioStream.cpp
OBJECTS=$(SOURCES:.cpp=.o)

TESTS=test_audio.cpp\
      test_deadlock.cpp
EXECUTABLES=$(TESTS:.cpp=)

all: $(OBJECTS) build

build:
	$(foreach src, $(TESTS), $(CXX) $(CFLAGS) -lc++ $(LIBRARIES) $(OBJECTS) $(src) -o $(src:.cpp=);)

.cpp.o:
	$(CXX) $(CFLAGS) -c $< -o $@

clean:
	rm $(EXECUTABLES) *.o

OwnedCriticalSection.h

// Reference: https://github.com/kinetiknz/cubeb/blob/master/src/cubeb_utils_unix.h

#ifndef OWNEDCRITICALSECTION_H
#define OWNEDCRITICALSECTION_H

#include <pthread.h>

/* This wraps a critical section to track the owner in ERRORCHECK mode. */
class OwnedCriticalSection
{
public:
  enum Mode
  {
    NORMAL,
    ERRORCHECK
  };

  OwnedCriticalSection(Mode aMode = NORMAL)
    : mMode(aMode)
  {
    pthread_mutexattr_t attr;
    pthread_mutexattr_init(&attr);
    pthread_mutexattr_settype(&attr, mMode == NORMAL ? PTHREAD_MUTEX_NORMAL :
                                                       PTHREAD_MUTEX_ERRORCHECK);
    assert(!pthread_mutex_init(&mMutex, &attr));
    pthread_mutexattr_destroy(&attr);
  }

  ~OwnedCriticalSection()
  {
    assert(!pthread_mutex_destroy(&mMutex));
  }

  void lock()
  {
    assert(!pthread_mutex_lock(&mMutex));
  }

  void unlock()
  {
    assert(!pthread_mutex_unlock(&mMutex));
  }

  void assertCurrentThreadOwns()
  {
    assert(mMode == ERRORCHECK);
    // EDEADLK: A deadlock condition was detected or the current thread
    // already owns the mutex.
    assert(pthread_mutex_lock(&mMutex) == EDEADLK);
  }

private:
  pthread_mutex_t mMutex;
  Mode mMode;

  // Disallow copy and assignment because pthread_mutex_t cannot be copied.
  OwnedCriticalSection(const OwnedCriticalSection&);
  OwnedCriticalSection& operator=(const OwnedCriticalSection&);
};

#endif /* OWNEDCRITICALSECTION_H */

test_audio.cpp

#include "AudioStream.h"
#include "utils.h"  // for delay
#include <math.h>   // for M_PI, sin
#include <vector>

const unsigned int kFequency = 44100;
const unsigned int kChannels = 2;

bool gCalled = false;

template<typename T> T ConvertSample(double aInput);
template<> float ConvertSample(double aInput) { return aInput; }
template<> short ConvertSample(double aInput) { return short(aInput * 32767.0f); }

class Synthesizer
{
public:
  Synthesizer(unsigned int aChannels, float aRate, double aVolume = 0.5)
    : mChannels(aChannels)
    , mRate(aRate)
    , mVolume(aVolume)
    , mPhase(std::vector<float>(aChannels, 0.0f))
  {}

  template<typename T>
  void Run(T* aBuffer, long aframes)
  {
    for (unsigned int i = 0; i < mChannels; ++i) {
      float increment = 2.0 * M_PI * GetFrequency(i) / mRate;
      for(long j = 0 ; j < aframes; ++j) {
        aBuffer[j * mChannels + i] = ConvertSample<T>(sin(mPhase[i]) * mVolume);
        mPhase[i] += increment;
      }
    }
  }

private:
  float GetFrequency(int aChannelIndex)
  {
    return 220.0f * (aChannelIndex + 1);
  }

  unsigned int mChannels;
  float mRate;
  double mVolume;
  std::vector<float> mPhase;
};

Synthesizer gSynthesizer(kChannels, kFequency);

/* AudioCallback */
template<typename T>
void callback(void* aBuffer, unsigned long aFrames)
{
  gSynthesizer.Run(static_cast<T*>(aBuffer), aFrames);
  gCalled = true;
}

int main()
{
  AudioStream as(AudioStream::Format::F32LE, kFequency, kChannels, callback<float>);
  // AudioStream as(AudioStream::Format::S16LE, kFequency, kChannels, callback<short>);

  as.Start();
  delay(1000);
  as.Stop();

  assert(gCalled && "Callback should be fired!");

  return 0;
}

test_deadlock.cpp

// Deadlock
//
// In CoreAudio, the ouput callback will holds a mutex shared with AudioUnit
// (hereinafter mutex_AU). Thus, if the callback requests another mutex M held
// by another thread, without releasing mutex_AU, then it will cause a
// deadlock when another thread holding the mutex M requests to use AudioUnit.
//
// The following figure illustrates the deadlock described above:
//
//    (Thread A)     holds
//  data_callback <---------- mutext_AudioUnit(mutex_AU)
//      |                            ^
//      |                            |
//      | request                    | request
//      |                            |
//      v           holds            |
//   mutex_M -------------------> Thread B

#include <assert.h>               // for assert
#include <pthread.h>              // for pthread
#include <signal.h>               // for signal
#include <unistd.h>               // for sleep, usleep

#include "AudioStream.h"          // for AudioStream
#include "utils.h"                // for LOG
#include "OwnedCriticalSection.h" // for OwnedCriticalSection

// The signal alias for calling our thread killer.
#define CALL_THREAD_KILLER SIGUSR1

const unsigned int kFequency = 44100;
const unsigned int kChannels = 2;

// If we apply ERRORCHECK mode, then we can't unlock a mutex locked by a
// different thread.
// OwnedCriticalSection gMutex(OwnedCriticalSection::Mode::ERRORCHECK);
OwnedCriticalSection gMutex;
using locker = std::lock_guard<OwnedCriticalSection>;

// Indicating whether the test is passed.
bool gPass = false;

// Indicating whether the data callback is fired.
bool gCalled = false;

// Indicating whether the data callback is running.
bool gCalling = false;

// Indicating whether the assigned task is done.
bool gTaskDone = false;

// Indicating whether our pending task thread is killed by ourselves.
bool gKilled = false;

void killer(int aSignal)
{
  assert(aSignal == CALL_THREAD_KILLER);
  LOG("pending task thread is killed!\n");
  gKilled = true;
}

uint64_t getThreadId(pthread_t aThread = NULL)
{
  uint64_t tid;
  // tid will be current thread id if aThread is null.
  pthread_threadid_np(aThread, &tid);
  return tid;
}

// The output callback fired from audio rendering mechanism, which is on
// out-of-main thread.
void callback(void* aBuffer, unsigned long aFrames)
{
  // The callback thread holds a mutex shared with AudioUnit.

  gCalling = true;

  uint64_t id = getThreadId();
  !gCalled && LOG("Output callback is on thread %llu, holding mutex_AU\n", id);
  gCalled = true;

  if (!gTaskDone) {
    // Force to switch threads by sleeping 10 ms. Notice that anything over
    // 10ms would produce a glitch. It's intended for testing deadlock,
    // so we ignore the fault here.
    LOG("[%llu] Force to switch threads\n", id);
    usleep(10000);
  }

  LOG("[%llu] Try getting another mutex: gMutex...\n", id);
  locker guard(gMutex);

  LOG("[%llu] Got mutex finally!\n", id);

  gCalling = false;
}

void* task(void*)
{
  // Hold the mutex.
  locker guard(gMutex);

  uint64_t id = getThreadId();
  LOG("Task thread: %llu, holding gMutex, is created\n", id);

  while(!gCalling) {
    LOG("[%llu] waiting for output callback before running task\n", id);
    usleep(1000); // Force to switch threads by sleeping 1 ms.
  }

  // Creating another AudioUnit when we already had one will cause a deadlock!
  LOG("[%llu] Try creating another AudioUnit (getting mutex_AU)...\n", id);
  AudioStream as(AudioStream::Format::F32LE, kFequency, kChannels, callback);

  LOG("[%llu] Another AudioUnit is created!\n", id);
  gTaskDone = true;

  return NULL;
}

// We provide one possible solution here:
// void* task(void*)
// {
//   uint64_t id = getThreadId();
//   LOG("Task thread: %llu is created\n", id);
//
//   while(!gCalling) {
//     LOG("[%llu] waiting for output callback before running task\n", id);
//     usleep(1000); // Force to switch threads by sleeping 1 ms.
//   }
//
//   // Creating another AudioUnit when we already had one will cause a deadlock!
//   LOG("[%llu] Try creating another AudioUnit (getting mutex_AU)...\n", id);
//   AudioStream as(AudioStream::Format::F32LE, kFequency, kChannels, callback);
//
//   LOG("[%llu] Another AudioUnit is created!\n", id);
//
//   // Hold the mutex.
//   LOG("[%llu] Try getting another mutex: gMutex...\n", id);
//   locker guard(gMutex);
//
//   LOG("[%llu] Got mutex finally!\n", id);
//
//   gTaskDone = true;
//
//   return NULL;
// }

void* watchdog(void* aSubject)
{
  uint64_t id = getThreadId();

  pthread_t subject = *((pthread_t *) aSubject);
  uint64_t sid = getThreadId(subject);

  LOG("Monitor thread %llu on thread %llu\n", sid, id);

  unsigned int sec = 1;
  LOG("[%llu] sleep %d seconds before checking task for thread %llu\n", id, sec, sid);
  sleep(sec); // Force to switch threads.

  if (!gTaskDone) {
    LOG("[%llu] Kill the task thread %llu!\n", id, sid);
    assert(!pthread_kill(subject, CALL_THREAD_KILLER));
    assert(!pthread_detach(subject));
    // The mutex held by the killed thread(subject) won't be released,
    // so we need unlock it manually. Notice that we can't unlock a mutex held
    // by other thread in OwnedCriticalSection::Mode::ERRORCHECK mode of gMutex.
    gMutex.unlock();
  }

  LOG("\n[%llu] Task is %sdone\n\n", id, gTaskDone ? "": "NOT ");
  gPass = gTaskDone;

  return NULL;
}

int main()
{
  AudioStream as(AudioStream::Format::F32LE, kFequency, kChannels, callback);

  // Install signal handler.
  signal(CALL_THREAD_KILLER, killer);

  pthread_t subject, detector;
  pthread_create(&subject, NULL, task, NULL);
  pthread_create(&detector, NULL, watchdog, (void *) &subject);

  as.Start();

  pthread_join(subject, NULL);
  pthread_join(detector, NULL);

  as.Stop();

  // If the callback is never fired, then the task must not be processed.
  // No need to keep checking in this case.
  assert(gCalled && "Callback should be fired!");

  // The task thread might keep running after the deadlock is freed, so we use
  // gPass instead of gTaskDone.
  assert(gPass && "Deadlock detected!");

  // False gPass implies there is a deadlock detected, so we need to kill the
  // pending task thread to free the deadlock and set gKilled to true.
  // True gPass means there is no deadlock and no need to kill any thread.
  assert(gPass != gKilled && "Killer is out of control!");

  return 0;
}

utils.h

#ifndef UTILS_H
#define UTILS_H

#include <iostream>
#include <time.h>

#define ENABLE_LOG true
#define LOG(...) ENABLE_LOG && fprintf(stderr, __VA_ARGS__)

void delay(unsigned int ms)
{
	clock_t end;
	end = clock() + ms * (CLOCKS_PER_SEC/1000);
	while (clock() < end) {}
}

#endif // #ifndef UTILS_H