Teensy 4.0 Arduino sketchy for TOSLINK (SPDIF) Input -> USB + SPDIF Output

PassThroughAsyncSpdifWithUSB.ino

// Teensy Library to stream Optical TOSLINK input simultaenously to two outputs:
// * USB Audio
// * SPDIF
//
//
// Test Setup:
// * Input: Google Chromecast Audio TOSLINK Optical @ 48 KHz -> PLR135/T10 -> Teensy SPDIF_IN
// * INput: USB serial console to set gain
// * Output: Teensy SPDIF_OUT -> Khadas Tone Board RCA SPDIF input -> Analog Speaker Amp
// * Output: USB -> Arch Linux (5.10 kernel) with Pulseaudio + Audacity
// * Clock: 600 MHz (11% CPU) and 150 MHz (44 MHz)
//
// PCB: https://github.com/kylemanna/teensy-toslink-receiver

#include <Audio.h>
#include <elapsedMillis.h>


// Order here is improtant because of some bug with the constructor?
// Declare and construct output first!
AudioOutputSPDIF3        spdifOut;
AsyncAudioInputSPDIF3    spdifIn(true, true, 100, 20);  // dither = true, noiseshaping = true, anti-aliasing attenuation=100dB, minimum resampling filter length=20

AudioAmplifier           amp1, amp2;
AudioConnection          patchCord3(spdifIn, 0, amp1, 0);
AudioConnection          patchCord4(spdifIn, 1, amp2, 0);

// Volume control for SPDIF
AudioConnection          patchCord1(amp1, 0, spdifOut, 0);
AudioConnection          patchCord2(amp2, 1, spdifOut, 1);

AudioOutputUSB           usb1;
AudioConnection          patchCord5(amp1, 0, usb1, 0);
AudioConnection          patchCord6(amp2, 0, usb1, 1);

void setup() {
  while (!Serial);
  AudioMemory(20); // Max observed was 14
}

void loop() {

  static elapsedMillis sincePrint;

  if (sincePrint > 500) {
    //double targetLatency = spdifIn.getTargetLantency();
    // Serial.print(", target: ");
    // Serial.println(targetLatency*1e6,2);
    Serial.print(">> buffer: ");
    Serial.print(spdifIn.getBufferedTime() * 1E3, 4);
    Serial.print(" ms, cpu usage: ");
    Serial.print(spdifIn.processorUsage());
    Serial.print("%, isLocked: ");
    Serial.print(spdifIn.isLocked());
    Serial.print(", frequency: ");
    Serial.print(spdifIn.getInputFrequency());
    Serial.print(" Hz, max mem usage: ");
    Serial.println(AudioMemoryUsageMax());
    
    sincePrint = 0;
  }

  while (Serial.available() > 0) {
    static char buf[64];
    static size_t buf_idx = 0;

    char c = Serial.read();
    if (c == '\n') {
      buf[buf_idx] = '\0';
      buf_idx = 0;

      auto f = String{buf}.toFloat();
      Serial.print(">> Setting gain to ");
      Serial.println(f);
      
      amp1.gain(f);
      amp2.gain(f);

    } else {
      buf[buf_idx] = c;
      buf_idx = (buf_idx + 1) % sizeof(buf);
    }
  }
}

Some remaining issue with USB audio creating noise as if there's a buffering or sample rate problem. Same problem exists with synthesized 1 kHz sine wave so. Audio is clear on the SPDIF output.