WebRTCPlayback.cs

using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine;
using Unity.WebRTC;
using UnityEngine.Networking;
using Unity.RenderStreaming;
using UnityEngine.UI;
using NativeWebSocket;

[System.Serializable]
public class StageJwt
{
  public string whip_url;
  public string[] active_participants;
  public static StageJwt CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<StageJwt>(jsonString);
  }
}

[System.Serializable]
public class StageParticipantUpdatedEvent
{
  public string session_id;
  public string event_name;
  public string user_id;
  public System.DateTime event_time;
  public string participant_id;
  public static StageParticipantUpdatedEvent CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<StageParticipantUpdatedEvent>(jsonString);
  }
}

[System.Serializable]
public class ChatAttributes
{
  public string @event;
  public StageParticipantUpdatedEvent particpantUpdatedEvent
  {
    set
    {
      particpantUpdatedEvent = value;
    }
    get
    {
      return JsonUtility.FromJson<StageParticipantUpdatedEvent>(@event);
    }
  }
  public static ChatAttributes CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatAttributes>(jsonString);
  }
}

[System.Serializable]
public class ChatMessage
{
  public string Type;
  public string EventName;
  public string Id;
  public string RequestId;
  public string Content;
  public ChatAttributes Attributes;
  public static ChatMessage CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatMessage>(jsonString);
  }
}

[System.Serializable]
public class ChatToken
{
  public System.DateTime sessionExpirationTime;
  public string token;
  public System.DateTime tokenExpirationTime;
  public static ChatToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatToken>(jsonString);
  }
}


[System.Serializable]
public class ParticipantToken
{
  public string token;
  public string participantId;
  public System.DateTime expirationTime;
  public static ParticipantToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ParticipantToken>(jsonString);
  }
}

[System.Serializable]
public class StageToken
{
  public ParticipantToken participantToken;
  public static StageToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<StageToken>(jsonString);
  }
}

[System.Serializable]
public class StageTokenRequestAttributes
{
  public string username;
  public StageTokenRequestAttributes(string username)
  {
    this.username = username;
  }
}

[System.Serializable]
public class StageTokenRequest
{
  public string stageArn;
  public string userId;
  public int duration;
  public StageTokenRequestAttributes attributes;
  public string[] capabilities;
  public StageTokenRequest(string stageArn, string userId, int duration, string[] capabilities, StageTokenRequestAttributes attributes)
  {
    this.stageArn = stageArn;
    this.userId = userId;
    this.duration = duration;
    this.capabilities = capabilities;
    this.attributes = attributes;
  }
}

[System.Serializable]
public class ChatTokenRequest
{
  public string chatArn;
  public string username;
  public string userId;
  public ChatTokenRequest(string chatArn, string username, string userId)
  {
    this.chatArn = chatArn;
    this.username = username;
    this.userId = userId;
  }
}

public class WebRTCPlayback : MonoBehaviour
{
  private MeshRenderer gameObject;
  RTCPeerConnection peerConnection;
  RawImage receiveImage;
  AudioSource receiveAudio;
  string participantId;
  WebSocket websocket;
  ParticipantToken participantToken;
  string whipUrl;
  bool isWhipped = false;

  private void Awake()
  {
    gameObject = this.gameObject;
  }
  async Task<ChatToken> GetChatToken()
  {
    using UnityWebRequest www = new UnityWebRequest("https://localhost:3000/chat-token");
    ChatTokenRequest tokenRequest = new ChatTokenRequest(
      "[YOUR CHAT ARN]",
      "IVS HUD Chat Demo User",
      System.Guid.NewGuid().ToString()
    );
    www.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(JsonUtility.ToJson(tokenRequest)));
    www.downloadHandler = new DownloadHandlerBuffer();
    www.method = UnityWebRequest.kHttpVerbPOST;
    www.SetRequestHeader("Content-Type", "application/json");
    var request = www.SendWebRequest();
    while (!request.isDone)
    {
      await Task.Yield();
    };
    var response = www.downloadHandler.text;
    if (www.result != UnityWebRequest.Result.Success)
    {
      Debug.Log(www.error);
      return default;
    }
    else
    {
      return ChatToken.CreateFromJSON(www.downloadHandler.text);
    }
  }

  async Task<WebSocket> ConnectChat()
  {
    var chatToken = await GetChatToken();
    websocket = new WebSocket("[YOUR WSS ENDPOINT]", chatToken.token);
    websocket.OnOpen += () =>
    {
      Debug.Log("Chat Connection: Open");
    };
    websocket.OnError += (e) =>
    {
      Debug.Log("Chat Connection: Error " + e);
    };
    websocket.OnClose += (e) =>
    {
      Debug.Log("Chat Connection: Closed");
    };
    websocket.OnMessage += (bytes) =>
    {
      var msgString = System.Text.Encoding.UTF8.GetString(bytes);
      Debug.Log("Chat Message Received! " + msgString);
      ChatMessage chatMsg = ChatMessage.CreateFromJSON(msgString);
      Debug.Log(chatMsg);
      if (chatMsg.Type == "EVENT" && chatMsg.EventName == "STAGE_PARTICIPANT_UPDATED")
      {
        if (chatMsg.Attributes.particpantUpdatedEvent.event_name == "Participant Published")
        {
          //receiveImage.gameObject.SetActive(true);
          participantId = chatMsg.Attributes.particpantUpdatedEvent.participant_id;
          Debug.Log("Participant ID: " + participantId);
          EstablishPeerConnection();
          StartCoroutine(DoWHIP());
        }
        else
        {
          receiveImage.texture = null;
          if (peerConnection != null)
          {
            peerConnection.Close();
            peerConnection.Dispose();
            peerConnection = null;
          }
        }
      }
    };
    return websocket;
  }

  async Task<StageToken> GetStageToken()
  {
    using UnityWebRequest www = new UnityWebRequest("http://localhost:3000/token");
    StageTokenRequest tokenRequest = new StageTokenRequest(
      "[YOUR STAGE ARN]",
      System.Guid.NewGuid().ToString(),
      1440,
      new string[] { "PUBLISH", "SUBSCRIBE" },
      new StageTokenRequestAttributes("ivs-rtx-hud-playback-demo")
    );
    www.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(JsonUtility.ToJson(tokenRequest)));
    www.downloadHandler = new DownloadHandlerBuffer();
    www.method = UnityWebRequest.kHttpVerbPOST;
    www.SetRequestHeader("Content-Type", "application/json");
    var request = www.SendWebRequest();
    while (!request.isDone)
    {
      await Task.Yield();
    };
    var response = www.downloadHandler.text;
    Debug.Log(response);
    if (www.result != UnityWebRequest.Result.Success)
    {
      Debug.Log(www.error);
      return default;
    }
    else
    {
      StageToken stageToken = StageToken.CreateFromJSON(www.downloadHandler.text);
      Debug.Log(stageToken);
      participantToken = stageToken.participantToken;

      // decode and parse token to get `whip_url`
      var parts = participantToken.token.Split('.');
      if (parts.Length > 2)
      {
        var decode = parts[1];
        var padLength = 4 - decode.Length % 4;
        if (padLength < 4)
        {
          decode += new string('=', padLength);
        }
        var bytes = System.Convert.FromBase64String(decode);
        var userInfo = System.Text.ASCIIEncoding.ASCII.GetString(bytes);
        StageJwt stageJwt = StageJwt.CreateFromJSON(userInfo);
        whipUrl = stageJwt.whip_url;
      }
      return stageToken;
    }
  }

  void EstablishPeerConnection()
  {
    peerConnection = new RTCPeerConnection();
    peerConnection.AddTransceiver(TrackKind.Audio);
    peerConnection.AddTransceiver(TrackKind.Video);
    peerConnection.OnIceConnectionChange = state => { Debug.Log(state); };

    Debug.Log("Adding Listeners");
    peerConnection.OnTrack = (RTCTrackEvent e) =>
    {
      Debug.Log("Remote OnTrack Called:");
      if (e.Track is VideoStreamTrack videoTrack)
      {
        videoTrack.OnVideoReceived += tex =>
        {
          Debug.Log("Video Recvd");
          receiveImage.texture = tex;
        };
      }
      if (e.Track is AudioStreamTrack audioTrack)
      {
        Debug.Log("Audio Recvd");
        receiveAudio.SetTrack(audioTrack);
        receiveAudio.loop = true;
        receiveAudio.Play();
      }
    };
  }

  async void Start()
  {
    Debug.Log("Start");
    receiveImage = GetComponent<RawImage>();
    receiveAudio = receiveImage.GetComponentInChildren<AudioSource>();
    StartCoroutine(WebRTC.Update());
    await ConnectChat();
    await websocket.Connect();
  }

  void Update()
  {
#if !UNITY_WEBGL || UNITY_EDITOR
    websocket?.DispatchMessageQueue();
#endif
  }

  IEnumerator DoWHIP()
  {
    Task getStageTokenTask = GetStageToken();
    yield return new WaitUntil(() => getStageTokenTask.IsCompleted);
    Debug.Log(participantToken.token);
    Debug.Log(participantToken.participantId);

    var offer = peerConnection.CreateOffer();
    yield return offer;

    var offerDesc = offer.Desc;
    var opLocal = peerConnection.SetLocalDescription(ref offerDesc);
    yield return opLocal;

    var filteredSdp = "";
    foreach (string sdpLine in offer.Desc.sdp.Split("\r\n"))
    {
      if (!sdpLine.StartsWith("a=extmap"))
      {
        filteredSdp += sdpLine + "\r\n";
      }
    }

    Debug.Log("Join?");
    using (UnityWebRequest publishRequest = new UnityWebRequest(whipUrl + "/subscribe/" + participantId))
    {
      publishRequest.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(filteredSdp));
      publishRequest.downloadHandler = new DownloadHandlerBuffer();
      publishRequest.method = UnityWebRequest.kHttpVerbPOST;
      publishRequest.SetRequestHeader("Content-Type", "application/sdp");
      publishRequest.SetRequestHeader("Authorization", "Bearer " + participantToken.token);
      yield return publishRequest.SendWebRequest();
      if (publishRequest.result != UnityWebRequest.Result.Success)
      {
        Debug.Log(publishRequest.error);
      }
      else
      {
        var answer = new RTCSessionDescription { type = RTCSdpType.Answer, sdp = publishRequest.downloadHandler.text };
        var opLocalRemote = peerConnection.SetRemoteDescription(ref answer);
        yield return opLocalRemote;
        if (opLocalRemote.IsError)
        {
          Debug.Log(opLocalRemote.Error);
        }
      }
    }
  }
  async void OnDestroy()
  {
    if (peerConnection != null)
    {
      peerConnection.Close();
      peerConnection.Dispose();
    }
    if (websocket != null) await websocket.Close();
  }
}