GStreamer Interaudio* Issue

README.md

This is an isolated example of a GStreamer audio issue that exists when using interaudio* elements.

• OS: NixOS 23.05.4076.8a4c17493e5c (Stoat) aarch64
• Kernel: 6.1.55
• CPU: Apple M1 Pro (Virt)
• Memory: 8.0GiB
• gst-launch-1.0 version: 1.22.5
• GStreamer: 1.22.5

Building

clang main.c `pkg-config gstreamer-1.0 --libs --cflags`

Reproduction

Execute the program and start the client pipeline (below). You will hear the usual audiotestsrc test tone. Try to start the second pipeline by entering a character in the terminal where the test program is running. The test tone begins to skip and slightly changes its pitch.

GST_DEBUG=3 gst-launch-1.0 udpsrc port=5000 ! \
  application/x-rtp,media=audio,encoding-name=OPUS,payload=96,clock-rate=48000 \
  ! rtpjitterbuffer ! rtpopusdepay ! opusdec ! audioconvert ! autoaudiosink

Note that when using tee instead of interaudio* everything works just as expected:

gst-launch-1.0 audiotestsrc is-live=true \
    ! audio/x-raw,format=S16LE,layout=interleaved,channels=2 \
    ! queue \
    ! tee name=t \
    t. ! queue ! audioconvert ! audioresample ! opusenc ! rtpopuspay ! udpsink host=127.0.0.1 port=5000 \
    t. ! queue ! audioconvert ! audioresample ! opusenc ! rtpopuspay ! udpsink host=127.0.0.1 port=5001

See main.py for a high-level demonstration.

main.c

#include <gst/gst.h>
#include <stdio.h>
#include <unistd.h>

#define LAUNCH_RTP \
	"interaudiosrc channel=audio0 ! " \
	"queue ! audioconvert ! audioresample ! opusenc ! rtpopuspay"

int main(int argc, char **argv) {
	GstElement *audioPipeline;
	GstElement *playbackPipeline;
	GstElement *playbackPipeline2;

	GError *error = NULL;

	gst_init(&argc, &argv);

	audioPipeline = gst_parse_launch(
		"audiotestsrc is-live=1 ! capsfilter "
		"caps=audio/x-raw,format=S16LE,layout=interleaved,channels=2 ! "
		"queue ! interaudiosink channel=audio0",
		&error);
	if (error != NULL) {
		g_print("Error creating audio pipeline: %s\n", error->message);
		g_error_free(error);
		return 1;
	}

	gchar *playbackPipelineStr = g_strconcat(LAUNCH_RTP, " ! udpsink host=127.0.0.1 port=5000", NULL);
	gchar *playbackPipelineStr2 = g_strconcat(LAUNCH_RTP, " ! udpsink host=127.0.0.1 port=5001", NULL);

	g_print("Playback pipeline: %s\n", playbackPipelineStr);
	playbackPipeline = gst_parse_launch(playbackPipelineStr, &error);
	if (error != NULL) {
		g_print("Error creating playback pipeline: %s\n", error->message);
		g_error_free(error);
		return 1;
	}

	g_print("Playback pipeline2: %s\n", playbackPipelineStr2);
	playbackPipeline2 = gst_parse_launch(playbackPipelineStr2, &error);
	if (error != NULL) {
		g_print("Error creating playback pipeline: %s\n", error->message);
		g_error_free(error);
		return 1;
	}
	
	g_free(playbackPipelineStr);
	g_free(playbackPipelineStr2);

	gst_element_set_state(audioPipeline, GST_STATE_PLAYING);
	sleep(1);

	gst_element_set_state(playbackPipeline, GST_STATE_PLAYING);
	

	g_print("Enter a character to start the second stream...\n");
	getchar();
	
	gst_element_set_state(playbackPipeline2, GST_STATE_PLAYING);

	g_print("Enter a character to stop the streams...\n");
	getchar();

	gst_element_set_state(audioPipeline, GST_STATE_NULL);
	gst_element_set_state(playbackPipeline, GST_STATE_NULL);
	gst_element_set_state(playbackPipeline2, GST_STATE_NULL);

	gst_object_unref(audioPipeline);
	gst_object_unref(playbackPipeline);
	gst_object_unref(playbackPipeline2);

  return 0;
}

main.py

import gi
import sys

gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject

# Initialize GStreamer
Gst.init(None)

pipeline1 = Gst.parse_launch("audiotestsrc is-live=true ! "
                             "audio/x-raw,format=S16LE,layout=interleaved,channels=2 ! "
                             "queue ! interaudiosink channel=audio0")

pipeline2 = Gst.parse_launch("interaudiosrc channel=audio0 ! "
                             "queue ! opusenc ! "
                             "rtpopuspay ! udpsink host=127.0.0.1 port=5000")

pipeline3 = Gst.parse_launch("interaudiosrc channel=audio0 ! "
                             "queue ! opusenc ! "
                             "rtpopuspay ! udpsink host=127.0.0.1 port=5001")

if not all([pipeline1, pipeline2, pipeline3]):
    sys.stderr.write("Error: Pipeline creation failed!\n")
    sys.exit(1)


for pipeline in [pipeline1, pipeline2]:
    pipeline.set_state(Gst.State.PLAYING)

Gst.debug_bin_to_dot_file(pipeline1, Gst.DebugGraphDetails.ALL, "pipeline-audio-src")
Gst.debug_bin_to_dot_file(pipeline2, Gst.DebugGraphDetails.ALL, "pipeline-net-0")
Gst.debug_bin_to_dot_file(pipeline3, Gst.DebugGraphDetails.ALL, "pipeline-net-1")

try:
    GObject.MainLoop().run()
except (KeyboardInterrupt, Exception) as e:
    print(f"Interrupted: {str(e)}")

for pipeline in [pipeline1, pipeline2, pipeline3]:
    pipeline.set_state(Gst.State.NULL)