using System; using System.Collections.Generic; using System.Runtime.InteropServices; using Agora.Rtc; using Agora.Util; using UnityEngine; using UnityEngine.Serialization; using UnityEngine.UI; using Logger = Agora.Util.Logger; namespace Agora_RTC_Plugin.API_Example.Examples.Advanced.PushEncodedVideoImage { public class PushEncodedVideoImage : MonoBehaviour { [FormerlySerializedAs("appIdInput")] [SerializeField] private AppIdInput _appIdInput; [Header("_____________Basic Configuration_____________")] [FormerlySerializedAs("APP_ID")] [SerializeField] public string _appID = ""; [FormerlySerializedAs("TOKEN")] [SerializeField] public string _token = ""; [FormerlySerializedAs("CHANNEL_NAME")] [SerializeField] public string _channelName = ""; public GameObject RolePrefab; private GameObject _roleLocal; public Text LogText; internal Logger Log; internal IRtcEngineEx RtcEngine = null; public Dictionary RolePositionDic = new Dictionary(); internal uint Uid1; internal uint Uid2; private System.Random _random = new System.Random(); private void Start() { LoadAssetData(); if (CheckAppId()) { Uid1 = (uint)(_random.Next() % 1000); Uid2 = (uint)(_random.Next() % 1000 + 1000); InitEngine(); JoinChannel1(); JoinChannel2(); } } [ContextMenu("ShowAgoraBasicProfileData")] private void LoadAssetData() { if (_appIdInput == null) return; _appID = _appIdInput.appID; _token = _appIdInput.token; _channelName = _appIdInput.channelName; } private bool CheckAppId() { Log = new Logger(LogText); return Log.DebugAssert(_appID.Length > 10, "Please fill in your appId in API-Example/profile/appIdInput.asset"); } private void InitEngine() { RtcEngine = Agora.Rtc.RtcEngine.CreateAgoraRtcEngineEx(); UserEventHandler handler = new UserEventHandler(this); RtcEngineContext context = new RtcEngineContext(_appID, 0, CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING, AUDIO_SCENARIO_TYPE.AUDIO_SCENARIO_GAME_STREAMING); RtcEngine.Initialize(context); RtcEngine.InitEventHandler(handler); RtcEngine.RegisterVideoEncodedFrameObserver(new VideoEncodedImageReceiver(this), OBSERVER_MODE.INTPTR); } private void JoinChannel1() { RtcEngine.SetClientRole(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER); RtcEngine.EnableVideo(); var option = new ChannelMediaOptions(); option.autoSubscribeVideo.SetValue(true); option.autoSubscribeAudio.SetValue(true); option.publishCameraTrack.SetValue(true); option.publishMicrophoneTrack.SetValue(true); option.clientRoleType.SetValue(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER); option.channelProfile.SetValue(CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING); var connection = new RtcConnection(); connection.channelId = _channelName; connection.localUid = Uid1; var nRet = RtcEngine.JoinChannelEx(_token, connection, option); this.Log.UpdateLog("joinChanne1: nRet" + nRet + " uid1:" + Uid1); } private void JoinChannel2() { RtcEngine.SetClientRole(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER); RtcEngine.EnableVideo(); SenderOptions senderOptions = new SenderOptions(); senderOptions.codecType = VIDEO_CODEC_TYPE.VIDEO_CODEC_GENERIC; RtcEngine.SetExternalVideoSource(true, true, EXTERNAL_VIDEO_SOURCE_TYPE.ENCODED_VIDEO_FRAME, senderOptions); var option = new ChannelMediaOptions(); option.autoSubscribeVideo.SetValue(true); option.autoSubscribeAudio.SetValue(true); option.publishCustomAudioTrack.SetValue(false); option.publishCameraTrack.SetValue(false); option.publishCustomVideoTrack.SetValue(false); option.publishEncodedVideoTrack.SetValue(true); option.clientRoleType.SetValue(CLIENT_ROLE_TYPE.CLIENT_ROLE_BROADCASTER); option.channelProfile.SetValue(CHANNEL_PROFILE_TYPE.CHANNEL_PROFILE_LIVE_BROADCASTING); var connection = new RtcConnection(); connection.channelId = _channelName; connection.localUid = Uid2; var nRet = RtcEngine.JoinChannelEx(_token, connection, option); this.Log.UpdateLog("joinChanne1: nRet" + nRet + " uid2:" + Uid2); } private void Update() { PermissionHelper.RequestMicrophontPermission(); PermissionHelper.RequestCameraPermission(); lock (this.RolePositionDic) { foreach (var e in this.RolePositionDic) { this.UpdateRolePositon(e.Key, e.Value); } } } private void OnDestroy() { Debug.Log("OnDestroy"); if (RtcEngine == null) return; RtcEngine.InitEventHandler(null); RtcEngine.UnRegisterVideoEncodedFrameObserver(); var connection = new RtcConnection(); connection.channelId = _channelName; connection.localUid = Uid1; RtcEngine.LeaveChannelEx(connection); connection.localUid = Uid2; RtcEngine.LeaveChannelEx(connection); RtcEngine.Dispose(); } public void CreateRole(string uid, bool isLocal) { if (GameObject.Find("Role" + uid) != null) { return; } var role = Instantiate(this.RolePrefab, this.transform); role.name = "Role" + uid; var text = role.transform.Find("Text").GetComponent(); text.text = uid; if (isLocal) { text.text += "\n(Local)"; role.AddComponent(); this._roleLocal = role; } else if (this._roleLocal != null) { var count = this.transform.childCount; this._roleLocal.transform.SetSiblingIndex(count - 1); } role.GetComponent().anchoredPosition = Vector3.zero; } public void DestroyRole(string uid, bool isLocal) { var name = "Role" + uid; var role = this.gameObject.transform.Find(name).gameObject; if (role) { Destroy(role); } if (isLocal) { this._roleLocal = null; } } private void UpdateRolePositon(string uid, Vector3 pos) { var name = "Role" + uid; var role = this.gameObject.transform.Find(name); if (role) { role.transform.localPosition = pos; } } public void StartPushEncodeVideoImage() { this.InvokeRepeating("UpdateForPushEncodeVideoImage", 0, 0.1f); this.Log.UpdateLog("Start PushEncodeVideoImage in every frame"); } public void StopPushEncodeVideoImage() { this.CancelInvoke("UpdateForPushEncodeVideoImage"); this.Log.UpdateLog("Stop PushEncodeVideoImage"); } internal string GetChannelName() { return _channelName; } private void UpdateForPushEncodeVideoImage() { //you can send any data not just video image byte if (_roleLocal) { //in this case, we send pos byte string json = JsonUtility.ToJson(this._roleLocal.transform.localPosition); byte[] data = System.Text.Encoding.Default.GetBytes(json); EncodedVideoFrameInfo encodedVideoFrameInfo = new EncodedVideoFrameInfo() { framesPerSecond = 60, codecType = VIDEO_CODEC_TYPE.VIDEO_CODEC_GENERIC, frameType = VIDEO_FRAME_TYPE_NATIVE.VIDEO_FRAME_TYPE_KEY_FRAME }; int nRet = this.RtcEngine.PushEncodedVideoImage(data, Convert.ToUInt32(data.Length), encodedVideoFrameInfo); Debug.Log("PushEncodedVideoImage: " + nRet); } } internal static void MakeVideoView(uint uid, string channelId = "") { var go = GameObject.Find(uid.ToString()); if (!ReferenceEquals(go, null)) { return; // reuse } // create a GameObject and assign to this new user var videoSurface = MakeImageSurface(uid.ToString()); if (ReferenceEquals(videoSurface, null)) return; // configure videoSurface if (uid == 0) { videoSurface.SetForUser(uid, channelId); } else { videoSurface.SetForUser(uid, channelId, VIDEO_SOURCE_TYPE.VIDEO_SOURCE_REMOTE); } videoSurface.OnTextureSizeModify += (int width, int height) => { float scale = (float)height / (float)width; videoSurface.transform.localScale = new Vector3(-5, 5 * scale, 1); Debug.Log("OnTextureSizeModify: " + width + " " + height); }; videoSurface.SetEnable(true); } // VIDEO TYPE 1: 3D Object private static VideoSurface MakePlaneSurface(string goName) { var go = GameObject.CreatePrimitive(PrimitiveType.Plane); if (go == null) { return null; } go.name = goName; // set up transform go.transform.Rotate(-90.0f, 0.0f, 0.0f); var yPos = UnityEngine.Random.Range(3.0f, 5.0f); var xPos = UnityEngine.Random.Range(-2.0f, 2.0f); go.transform.position = Vector3.zero; go.transform.localScale = new Vector3(0.25f, 0.5f, 0.5f); // configure videoSurface var videoSurface = go.AddComponent(); return videoSurface; } // Video TYPE 2: RawImage private static VideoSurface MakeImageSurface(string goName) { GameObject go = new GameObject(); if (go == null) { return null; } go.name = goName; // to be renderered onto go.AddComponent(); // make the object draggable go.AddComponent(); var canvas = GameObject.Find("VideoCanvas"); if (canvas != null) { go.transform.parent = canvas.transform; Debug.Log("add video view"); } else { Debug.Log("Canvas is null video view"); } // set up transform go.transform.Rotate(0f, 0.0f, 180.0f); go.transform.localPosition = Vector3.zero; go.transform.localScale = new Vector3(2f, 3f, 1f); // configure videoSurface var videoSurface = go.AddComponent(); return videoSurface; } internal static void DestroyVideoView(uint uid) { var go = GameObject.Find(uid.ToString()); if (!ReferenceEquals(go, null)) { Destroy(go); } } } #region -- Agora Event --- internal class UserEventHandler : IRtcEngineEventHandler { private readonly PushEncodedVideoImage _pushEncodedVideoImage; internal UserEventHandler(PushEncodedVideoImage videoSample) { _pushEncodedVideoImage = videoSample; } public override void OnError(int err, string msg) { _pushEncodedVideoImage.Log.UpdateLog(string.Format("OnError err: {0}, msg: {1}", err, msg)); } public override void OnJoinChannelSuccess(RtcConnection connection, int elapsed) { int build = 0; Debug.Log("Agora: OnJoinChannelSuccess "); _pushEncodedVideoImage.Log.UpdateLog(string.Format("sdk version: ${0}", _pushEncodedVideoImage.RtcEngine.GetVersion(ref build))); _pushEncodedVideoImage.Log.UpdateLog( string.Format("OnJoinChannelSuccess channelName: {0}, uid: {1}, elapsed: {2}", connection.channelId, connection.localUid, elapsed)); if (connection.localUid >= 1000) { _pushEncodedVideoImage.CreateRole(connection.localUid.ToString(), true); _pushEncodedVideoImage.Log.UpdateLog("you can drag your role to every where"); _pushEncodedVideoImage.StartPushEncodeVideoImage(); } else { PushEncodedVideoImage.MakeVideoView(0); } } public override void OnRejoinChannelSuccess(RtcConnection connection, int elapsed) { _pushEncodedVideoImage.Log.UpdateLog("OnRejoinChannelSuccess"); } public override void OnLeaveChannel(RtcConnection connection, RtcStats stats) { _pushEncodedVideoImage.Log.UpdateLog("OnLeaveChannel"); if (connection.localUid >= 1000) { _pushEncodedVideoImage.DestroyRole(connection.localUid.ToString(), true); _pushEncodedVideoImage.StopPushEncodeVideoImage(); } else { PushEncodedVideoImage.DestroyVideoView(0); } } public override void OnClientRoleChanged(RtcConnection connection, CLIENT_ROLE_TYPE oldRole, CLIENT_ROLE_TYPE newRole) { _pushEncodedVideoImage.Log.UpdateLog("OnClientRoleChanged"); } public override void OnUserJoined(RtcConnection connection, uint uid, int elapsed) { _pushEncodedVideoImage.Log.UpdateLog(string.Format("OnUserJoined uid: ${0} elapsed: ${1}", uid, elapsed)); if (uid == _pushEncodedVideoImage.Uid1 || uid == _pushEncodedVideoImage.Uid2) return; if (uid >= 1000) { _pushEncodedVideoImage.CreateRole(uid.ToString(), false); //you must set options.encodedFrameOnly = true when you receive other VideoSubscriptionOptions options = new VideoSubscriptionOptions(); options.encodedFrameOnly.SetValue(true); int nRet = _pushEncodedVideoImage.RtcEngine.SetRemoteVideoSubscriptionOptionsEx(uid, options, connection); _pushEncodedVideoImage.Log.UpdateLog("SetRemoteVideoSubscriptionOptions nRet:" + nRet); } else { PushEncodedVideoImage.MakeVideoView(uid, _pushEncodedVideoImage.GetChannelName()); } } public override void OnUserOffline(RtcConnection connection, uint uid, USER_OFFLINE_REASON_TYPE reason) { _pushEncodedVideoImage.Log.UpdateLog(string.Format("OnUserOffLine uid: ${0}, reason: ${1}", uid, (int)reason)); if (uid == _pushEncodedVideoImage.Uid1 || uid == _pushEncodedVideoImage.Uid2) return; if (uid >= 1000) { _pushEncodedVideoImage.DestroyRole(uid.ToString(), false); } else { PushEncodedVideoImage.DestroyVideoView(uid); } } public override void OnChannelMediaRelayEvent(int code) { _pushEncodedVideoImage.Log.UpdateLog(string.Format("OnChannelMediaRelayEvent: {0}", code)); } public override void OnChannelMediaRelayStateChanged(int state, int code) { _pushEncodedVideoImage.Log.UpdateLog(string.Format("OnChannelMediaRelayStateChanged state: {0}, code: {1}", state, code)); } } internal class VideoEncodedImageReceiver : IVideoEncodedFrameObserver { private readonly PushEncodedVideoImage _pushEncodedVideoImage; internal VideoEncodedImageReceiver(PushEncodedVideoImage videoSample) { _pushEncodedVideoImage = videoSample; } public override bool OnEncodedVideoFrameReceived(uint uid, IntPtr imageBufferPtr, UInt64 length, EncodedVideoFrameInfo videoEncodedFrameInfo) { byte[] imageBuffer = new byte[length]; Marshal.Copy(imageBufferPtr, imageBuffer, 0, (int)length); string str = System.Text.Encoding.Default.GetString(imageBuffer); var pos = JsonUtility.FromJson(str); var uidStr = uid.ToString(); Debug.Log("OnEncodedVideoImageReceived" + uid + " pos" + str); //this called is not in Unity MainThread.we need push data in this dic. And read it in Update() lock (_pushEncodedVideoImage.RolePositionDic) { if (_pushEncodedVideoImage.RolePositionDic.ContainsKey(uidStr)) { _pushEncodedVideoImage.RolePositionDic[uidStr] = pos; } else { _pushEncodedVideoImage.RolePositionDic.Add(uidStr, pos); } } return true; } } #endregion }