This script is an example of how to create dynamic user controlled cameras and modify the environment and objectives of a Unity game that is live streamed to Amazon IVS.

WebRTCPublish.cs

using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine;
using Unity.WebRTC;
using UnityEngine.Networking;
using Cinemachine;
using NativeWebSocket;

[System.Serializable]
public class ChatAttributes
{
  public string username;
  public static ChatAttributes CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatAttributes>(jsonString);
  }
}

[System.Serializable]
public class ChatSender
{
  public string UserId;
  public ChatAttributes Attributes;
  public static ChatSender CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatSender>(jsonString);
  }
}

[System.Serializable]
public class ChatMessage
{
  public string Type;
  public string Id;
  public string RequestId;
  public string Content;
  public ChatSender Sender;
  public static ChatMessage CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatMessage>(jsonString);
  }
}

[System.Serializable]
public class ChatToken
{
  public System.DateTime sessionExpirationTime;
  public string token;
  public System.DateTime tokenExpirationTime;
  public static ChatToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ChatToken>(jsonString);
  }
}

[System.Serializable]
public class ParticipantToken
{
  public string token;
  public string participantId;
  public System.DateTime expirationTime;
  public static ParticipantToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<ParticipantToken>(jsonString);
  }
}

[System.Serializable]
public class StageToken
{
  public ParticipantToken participantToken;
  public static StageToken CreateFromJSON(string jsonString)
  {
    return JsonUtility.FromJson<StageToken>(jsonString);
  }
}

[RequireComponent(typeof(AudioListener))]

[System.Serializable]
public class StageTokenRequestAttributes
{
  public string username;
  public StageTokenRequestAttributes(string username)
  {
    this.username = username;
  }
}

[System.Serializable]
public class StageTokenRequest
{
  public string stageArn;
  public string userId;
  public int duration;
  public StageTokenRequestAttributes attributes;
  public string[] capabilities;
  public StageTokenRequest(string stageArn, string userId, int duration, string[] capabilities, StageTokenRequestAttributes attributes)
  {
    this.stageArn = stageArn;
    this.userId = userId;
    this.duration = duration;
    this.capabilities = capabilities;
    this.attributes = attributes;
  }
}

[System.Serializable]
public class ChatTokenRequest
{
  public string chatArn;
  public string username;
  public string userId;
  public ChatTokenRequest(string chatArn, string username, string userId)
  {
    this.chatArn = chatArn;
    this.username = username;
    this.userId = userId;
  }
}

public class WebRTCPublish : MonoBehaviour
{
  RTCPeerConnection peerConnection;
  MediaStreamTrack videoTrack;
  public AudioStreamTrack audioTrack;
  Camera cam;
  CinemachineVirtualCamera virtualCamera;

  ParticipantToken participantToken;
  WebSocket websocket;
  TimeManager timeManager;
  public GameObject jump;
  public GameObject kart;
  async Task<ChatToken> GetChatToken()
  {
    using UnityWebRequest www = new UnityWebRequest("http://localhost:3000/chat-token");
    ChatTokenRequest tokenRequest = new ChatTokenRequest(
      "[YOUR CHAT ROOM ARN]",
      "IVS Broadcast Dynamic Cam Chat Demo User",
      System.Guid.NewGuid().ToString()
    );
    www.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(JsonUtility.ToJson(tokenRequest)));
    www.downloadHandler = new DownloadHandlerBuffer();
    www.method = UnityWebRequest.kHttpVerbPOST;
    www.SetRequestHeader("Content-Type", "application/json");
    var request = www.SendWebRequest();
    while (!request.isDone)
    {
      await Task.Yield();
    };
    var response = www.downloadHandler.text;
    if (www.result != UnityWebRequest.Result.Success)
    {
      Debug.Log(www.error);
      return default;
    }
    else
    {
      return ChatToken.CreateFromJSON(www.downloadHandler.text);
    }
  }

  async Task<WebSocket> ConnectChat()
  {
    var chatToken = await GetChatToken();
    websocket = new WebSocket("[YOUR WSS ENDPOINT]", chatToken.token);
    websocket.OnOpen += () =>
    {
      Debug.Log("Chat Connection: Open");
    };
    websocket.OnError += (e) =>
    {
      Debug.Log("Chat Connection: Error " + e);
    };
    websocket.OnClose += (e) =>
    {
      Debug.Log("Chat Connection: Closed");
    };
    websocket.OnMessage += (bytes) =>
    {
      var msgString = System.Text.Encoding.UTF8.GetString(bytes);
      Debug.Log("Chat Message Received! " + msgString);
      ChatMessage chatMsg = ChatMessage.CreateFromJSON(msgString);
      Debug.Log(chatMsg);
      var body = virtualCamera.GetCinemachineComponent<CinemachineTransposer>();
      float currentX = body.m_FollowOffset[0];
      float currentY = body.m_FollowOffset[1];
      float currentZ = body.m_FollowOffset[2];
      if (chatMsg.Type == "MESSAGE")
      {
        if (chatMsg.Content.ToLower() == "jump")
        {
          Vector3 kartPos = kart.transform.position;
          Vector3 kartDirection = kart.transform.forward;
          Quaternion kartRotation = kart.transform.rotation;
          float spawnDistance = 10;
          Vector3 spawnPos = kartPos + kartDirection * spawnDistance;
          Instantiate(jump, spawnPos, kartRotation);
        }
        if (chatMsg.Content.ToLower() == "+10")
        {
          timeManager.AdjustTime(10f);
        }
        if (chatMsg.Content.ToLower() == "-10")
        {
          timeManager.AdjustTime(-10f);
        }
        if (chatMsg.Content.ToLower() == "up")
        {
          body.m_FollowOffset = new Vector3(currentX, currentY + 0.5f, currentZ);
        }
        if (chatMsg.Content.ToLower() == "down")
        {
          body.m_FollowOffset = new Vector3(currentX, currentY - 0.5f, currentZ);
        }
        if (chatMsg.Content.ToLower() == "left")
        {
          body.m_FollowOffset = new Vector3(currentX - 0.5f, currentY, currentZ);
        }
        if (chatMsg.Content.ToLower() == "right")
        {
          body.m_FollowOffset = new Vector3(currentX + 0.5f, currentY, currentZ);
        }
        if (chatMsg.Content.ToLower() == "out")
        {
          body.m_FollowOffset = new Vector3(currentX, currentY, currentZ - 0.5f);
        }
        if (chatMsg.Content.ToLower() == "in")
        {
          body.m_FollowOffset = new Vector3(currentX, currentY, currentZ + 0.5f);
        }
      }
    };
    return websocket;
  }

  async Task<StageToken> GetStageToken()
  {
    using UnityWebRequest www = new UnityWebRequest("http://localhost:3000/token");
    StageTokenRequest tokenRequest = new StageTokenRequest(
      "[YOUR STAGE ARN]",
      System.Guid.NewGuid().ToString(),
      1440,
      new string[] { "PUBLISH", "SUBSCRIBE" },
      new StageTokenRequestAttributes("ivs-rtx-broadcast-dynamic-cam-demo")
    );
    www.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(JsonUtility.ToJson(tokenRequest)));
    www.downloadHandler = new DownloadHandlerBuffer();
    www.method = UnityWebRequest.kHttpVerbPOST;
    www.SetRequestHeader("Content-Type", "application/json");
    var request = www.SendWebRequest();
    while (!request.isDone)
    {
      await Task.Yield();
    };
    var response = www.downloadHandler.text;
    Debug.Log(response);
    if (www.result != UnityWebRequest.Result.Success)
    {
      Debug.Log(www.error);
      return default;
    }
    else
    {
      StageToken stageToken = StageToken.CreateFromJSON(www.downloadHandler.text);
      Debug.Log(stageToken);
      participantToken = stageToken.participantToken;
      return stageToken;
    }
  }
  async void Start()
  {
    Debug.Log(kart.transform.position);
    StartCoroutine(WebRTC.Update());
    peerConnection = new RTCPeerConnection
    {
      OnIceConnectionChange = state => { Debug.Log("Peer Connection: " + state); }
    };
    timeManager = GameObject.FindObjectOfType(typeof(TimeManager)) as TimeManager;
    cam = GetComponent<Camera>();
    virtualCamera = cam.transform.parent.GetComponent<CinemachineVirtualCamera>();
    videoTrack = cam.CaptureStreamTrack(1280, 720);
    peerConnection.AddTrack(videoTrack);
    AudioListener audioListener = cam.GetComponent<AudioListener>();
    audioTrack = new AudioStreamTrack(audioListener) { Loopback = true };
    peerConnection.AddTrack(audioTrack);
    StartCoroutine(DoWHIP());
    await ConnectChat();
    await websocket.Connect();
  }

  void Update()
  {
#if !UNITY_WEBGL || UNITY_EDITOR
    websocket?.DispatchMessageQueue();
#endif
  }

  IEnumerator DoWHIP()
  {
    Task getStageTokenTask = GetStageToken();
    yield return new WaitUntil(() => getStageTokenTask.IsCompleted);
    Debug.Log(participantToken.token);
    Debug.Log(participantToken.participantId);

    var offer = peerConnection.CreateOffer();
    yield return offer;

    var offerDesc = offer.Desc;
    var opLocal = peerConnection.SetLocalDescription(ref offerDesc);
    yield return opLocal;

    var filteredSdp = "";
    foreach (string sdpLine in offer.Desc.sdp.Split("\r\n"))
    {
      if (!sdpLine.StartsWith("a=extmap"))
      {
        filteredSdp += sdpLine + "\r\n";
      }
    }
    using (UnityWebRequest www = new UnityWebRequest("https://global.whip.live-video.net/"))
    {
      www.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.ASCII.GetBytes(filteredSdp));
      www.downloadHandler = new DownloadHandlerBuffer();
      www.method = UnityWebRequest.kHttpVerbPOST;
      www.SetRequestHeader("Content-Type", "application/sdp");
      www.SetRequestHeader("Authorization", "Bearer " + participantToken.token);
      yield return www.SendWebRequest();
      if (www.result != UnityWebRequest.Result.Success)
      {
        Debug.Log(www.error);
      }
      else
      {
        var answer = new RTCSessionDescription { type = RTCSdpType.Answer, sdp = www.downloadHandler.text };
        var opRemote = peerConnection.SetRemoteDescription(ref answer);
        yield return opRemote;
        if (opRemote.IsError)
        {
          Debug.Log(opRemote.Error);
        }
      }
    }
  }

  async void OnDestroy()
  {
    Debug.Log("OnDestroy");
    peerConnection.Close();
    peerConnection.Dispose();
    if (videoTrack != null) videoTrack.Dispose();
    if (audioTrack != null) audioTrack.Dispose();
    if (websocket != null) await websocket.Close();
  }
}