| // | |
| // AudioTapMixer.swift | |
| // | |
| // | |
| // Created by Jack Youstra on 7/20/23. All rights reserved. | |
| // | |
| import AVFoundation | |
| struct ChangedAudioTapContext { | |
| var format: AVAudioFormat | |
| let makeDistortion: () -> AVAudioUnitEffect | |
| var sampleCount: Float64 = 0 | |
| struct PerFrame { | |
| let sourceAudioBufferList: UnsafePointer<AudioBufferList> | |
| let frameCount: AVAudioFrameCount | |
| } | |
| struct PerPreparation { | |
| let engine: AVAudioEngine | |
| let outputBuffer: AVAudioPCMBuffer | |
| } | |
| var perFrame: PerFrame! | |
| var perPreparation: PerPreparation! | |
| } | |
| public extension AVMutableAudioMixInputParameters { | |
| static func effect(track: AVAssetTrack, effect makeDistortion: @escaping () -> AVAudioUnitEffect) async throws -> AVMutableAudioMixInputParameters { | |
| let param = Self(track: track) | |
| let audioDescription: [CMAudioFormatDescription] = try await track.load(.formatDescriptions) | |
| assert(audioDescription.count == 1) | |
| let format = AVAudioFormat(cmAudioFormatDescription: audioDescription.first!) | |
| let clientInfoPointer = UnsafeMutablePointer<ChangedAudioTapContext>.allocate(capacity: 1) | |
| clientInfoPointer.pointee = ChangedAudioTapContext(format: format, makeDistortion: makeDistortion) | |
| var callbacks = MTAudioProcessingTapCallbacks(version: kMTAudioProcessingTapCallbacksVersion_0, clientInfo: clientInfoPointer) | |
| { _, clientInfo, tapStorageOut in | |
| // initialize | |
| tapStorageOut.pointee = clientInfo! | |
| } finalize: { tap in | |
| // clean up | |
| let clientInfo = MTAudioProcessingTapGetStorage(tap) | |
| clientInfo.deallocate() | |
| } prepare: { tap, maxFrames, processingFormat in | |
| // allocate memory for sound processing | |
| assert((processingFormat.pointee.mFormatFlags & kAudioFormatFlagIsFloat) != 0) | |
| assert((processingFormat.pointee.mFormatID & kAudioFormatLinearPCM) != 0) | |
| let format: AVAudioCommonFormat | |
| if Int(processingFormat.pointee.mFormatFlags) & AudioFormatFlags.bitWidth == 32 { | |
| format = .pcmFormatFloat32 | |
| } else if Int(processingFormat.pointee.mFormatFlags) & AudioFormatFlags.bitWidth == 64 { | |
| format = .pcmFormatFloat64 | |
| } else { | |
| fatalError() | |
| } | |
| let changedFormat = AVAudioFormat(streamDescription: processingFormat)! | |
| let clientInfoPointer = MTAudioProcessingTapGetStorage(tap).assumingMemoryBound(to: ChangedAudioTapContext.self) | |
| clientInfoPointer.pointee.format = changedFormat | |
| let client = clientInfoPointer.pointee | |
| assert(client.format.commonFormat == format) | |
| let engine = AVAudioEngine() | |
| let srcNode = AVAudioSourceNode { _, _, innerFrameCount, audioBufferList -> OSStatus in | |
| let frame = clientInfoPointer.pointee.perFrame! | |
| assert(frame.frameCount == innerFrameCount) | |
| assert(innerFrameCount <= maxFrames) | |
| // TODO: Remove the copy here and just change the pointer | |
| // if you find some clever way to do so | |
| audioBufferList.pointee = frame.sourceAudioBufferList.pointee | |
| return noErr | |
| } | |
| let maximumFrames = AVAudioFrameCount(maxFrames) | |
| if engine.isRunning { engine.stop() } | |
| try! engine.enableManualRenderingMode(.realtime, format: client.format, maximumFrameCount: maximumFrames) | |
| let mainMixer = engine.mainMixerNode | |
| let output = engine.outputNode | |
| let outputFormat = output.inputFormat(forBus: 0) | |
| let sampleRate = Float(outputFormat.sampleRate) | |
| let inputFormat = client.format | |
| engine.attach(srcNode) | |
| let distortion = client.makeDistortion() | |
| // Can't be interleaved | |
| // https://developer.apple.com/documentation/audiotoolbox/auaudiounitbus/1387644-setformat | |
| // idk how to deinterleave??? | |
| // assert(!inputFormat.isInterleaved) | |
| // assert(!outputFormat.isInterleaved) | |
| // engine.connect(srcNode, to: mainMixer, format: inputFormat) | |
| engine.connect(srcNode, to: distortion, format: inputFormat) | |
| engine.connect(distortion, to: mainMixer, format: inputFormat) | |
| engine.connect(mainMixer, to: output, format: outputFormat) | |
| mainMixer.outputVolume = 1.0 | |
| try! engine.start() | |
| let outputBuffer = AVAudioPCMBuffer(pcmFormat: client.format, frameCapacity: maximumFrames)! | |
| clientInfoPointer.pointee.perPreparation = .init(engine: engine, outputBuffer: outputBuffer) | |
| } unprepare: { tap in | |
| // deallocate memory for sound processing | |
| let clientInfoPointer = MTAudioProcessingTapGetStorage(tap).assumingMemoryBound(to: ChangedAudioTapContext.self) | |
| clientInfoPointer.pointee.perPreparation = nil | |
| } process: { tap, numberFrames, _, bufferListInOut, numberFramesOut, flagsOut in | |
| let clientPtr = MTAudioProcessingTapGetStorage(tap).assumingMemoryBound(to: ChangedAudioTapContext.self) | |
| guard noErr == MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut) else { | |
| assertionFailure() | |
| return | |
| } | |
| assert(clientPtr.pointee.format.commonFormat == .pcmFormatFloat32) | |
| let frameCount = AVAudioFrameCount(numberFramesOut.pointee) | |
| clientPtr.pointee.perFrame = .init(sourceAudioBufferList: bufferListInOut, frameCount: frameCount) | |
| clientPtr.pointee.perPreparation.outputBuffer.frameLength = frameCount | |
| let code: UnsafeMutablePointer<OSStatus> = .allocate(capacity: 1) | |
| defer { | |
| code.deallocate() | |
| } | |
| let result = clientPtr.pointee.perPreparation.engine.manualRenderingBlock(frameCount, clientPtr.pointee.perPreparation.outputBuffer.mutableAudioBufferList, code) | |
| assert(result == .success, "manualRenderingBlock failed: \(result), \(code.pointee)") | |
| bufferListInOut.pointee = clientPtr.pointee.perPreparation.outputBuffer.mutableAudioBufferList.pointee | |
| } | |
| var tap: Unmanaged<MTAudioProcessingTap>? | |
| guard MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) == noErr else { | |
| preconditionFailure() | |
| } | |
| param.audioTapProcessor = tap?.takeUnretainedValue() | |
| tap?.release() | |
| return param | |
| } | |
| } |