/****************************************************************************
* Copyright 2019 Nreal Techonology Limited. All rights reserved.
*
* This file is part of NRSDK.
*
* https://www.nreal.ai/
*
*****************************************************************************/
namespace NRKernal
{
using AOT;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
using UnityEngine;
public interface IFrameProcessor
{
UInt64 GetFrameHandle();
UInt64 GetViewPortListHandle();
void SubmitFrame(ulong frameHandle, ulong viewPortListHandle);
void Update();
void Initialize(NativeRenderring nativeRenderer);
void Destroy();
}
///
/// NRNativeRender operate rendering-related things, provides the feature of optimized rendering
/// and low latency.
[ScriptOrder(NativeConstants.NRRENDER_ORDER)]
public class NRRenderer : MonoBehaviour
{
/// Renders the event delegate described by eventID.
/// Identifier for the event.
private delegate void RenderEventDelegate(int eventID);
/// Handle of the render thread.
private static RenderEventDelegate RenderThreadHandle = new RenderEventDelegate(RunOnRenderThread);
/// The render thread handle pointer.
private static IntPtr RenderThreadHandlePtr = Marshal.GetFunctionPointerForDelegate(RenderThreadHandle);
private const int SETRENDERTEXTUREEVENT = 0x0001;
private const int STARTNATIVERENDEREVENT = 0x0002;
private const int RESUMENATIVERENDEREVENT = 0x0003;
private const int PAUSENATIVERENDEREVENT = 0x0004;
// private const int STOPNATIVERENDEREVENT = 0x0005;
private const int SUBMIT_EVENT = 0x0006;
//When application resumed, frame data must be updated in Update before submited in RenderCoroutine.
private bool frameReady = false;
public enum Eyes
{
/// Left Display.
Left = 0,
/// Right Display.
Right = 1,
Count = 2
}
public Camera leftCamera;
public Camera rightCamera;
/// Gets or sets the native renderring.
/// The m native renderring.
private static NativeRenderring m_NativeRenderring;
static NativeRenderring NativeRenderring
{
get
{
if (NRSessionManager.Instance.NativeAPI != null)
{
m_NativeRenderring = NRSessionManager.Instance.NativeAPI.NativeRenderring;
}
else if (m_NativeRenderring == null)
{
m_NativeRenderring = new NativeRenderring();
}
return m_NativeRenderring;
}
set
{
m_NativeRenderring = value;
}
}
/// The scale factor.
public static float ScaleFactor = 1f;
private const float m_DefaultFocusDistance = 1.4f;
private float m_FocusDistance = 1.4f;
public float FocusDistance
{
get { return m_FocusDistance; }
}
private static int _TextureBufferSize = 4;
/// Number of eye textures.
private static int EyeTextureCount = _TextureBufferSize * (int)Eyes.Count;
/// The eye textures.
private RenderTexture[] eyeTextures;
/// Dictionary of rights.
private Dictionary m_RTDict = new Dictionary();
/// Frame process.
private IFrameProcessor m_FrameProcessor;
/// Values that represent renderer states.
public enum RendererState
{
UnInitialized,
Initialized,
Running,
Paused,
Destroyed
}
private bool m_IsTrackChanging = false;
private RenderTexture m_HijackRenderTextureLeft;
private RenderTexture m_HijackRenderTextureRight;
/// The current state.
private RendererState m_CurrentState = RendererState.UnInitialized;
/// Gets the current state.
/// The current state.
public RendererState CurrentState
{
get
{
return m_CurrentState;
}
set
{
m_CurrentState = value;
}
}
public NRTrackingModeChangedListener TrackingLostListener
{
get
{
return NRSessionManager.Instance.TrackingLostListener;
}
}
#if !UNITY_EDITOR
private int currentEyeTextureIdx = 0;
private int nextEyeTextureIdx = 0;
#endif
/// Gets a value indicating whether this object is linear color space.
/// True if this object is linear color space, false if not.
public static bool isLinearColorSpace
{
get
{
return QualitySettings.activeColorSpace == ColorSpace.Linear;
}
}
/// Initialize the render pipleline.
/// Left Eye.
/// Right Eye.
///
/// ### provide the pose of camera every frame.
public void Initialize(Camera leftcamera, Camera rightcamera)
{
NRDebugger.Info("[NRRender] Initialize");
if (m_CurrentState != RendererState.UnInitialized)
{
return;
}
leftCamera = leftcamera;
rightCamera = rightcamera;
#if !UNITY_EDITOR
leftCamera.depthTextureMode = DepthTextureMode.None;
rightCamera.depthTextureMode = DepthTextureMode.None;
leftCamera.rect = new Rect(0, 0, 1, 1);
rightCamera.rect = new Rect(0, 0, 1, 1);
leftCamera.enabled = false;
rightCamera.enabled = false;
m_CurrentState = RendererState.Initialized;
StartCoroutine(StartUp());
#endif
if (TrackingLostListener != null)
{
TrackingLostListener.OnTrackStateChanged += OnTrackStateChanged;
}
}
private void OnTrackStateChanged(bool trackChanging, RenderTexture leftRT, RenderTexture rightRT)
{
if (trackChanging)
{
m_IsTrackChanging = true;
m_HijackRenderTextureLeft = leftRT;
m_HijackRenderTextureRight = rightRT;
}
else
{
m_IsTrackChanging = false;
m_HijackRenderTextureLeft = null;
m_HijackRenderTextureRight = null;
}
}
/// Prepares this object for use.
/// An IEnumerator.
private IEnumerator StartUp()
{
var virtualDisplay = GameObject.FindObjectOfType();
while (virtualDisplay == null || !virtualDisplay.Subsystem.running)
{
NRDebugger.Info("[NRRender] Wait virtual display ready...");
yield return new WaitForEndOfFrame();
if (virtualDisplay == null)
{
virtualDisplay = GameObject.FindObjectOfType();
}
}
yield return new WaitForEndOfFrame();
yield return new WaitForEndOfFrame();
yield return new WaitForEndOfFrame();
NRDebugger.Info("[NRRender] StartUp");
#if !UNITY_EDITOR
NativeRenderring.Create();
StartCoroutine(RenderCoroutine());
#endif
GL.IssuePluginEvent(RenderThreadHandlePtr, STARTNATIVERENDEREVENT);
if (m_FrameProcessor == null)
CreateRenderTextures();
NRDebugger.Info("[NRRender] StartUp Finish");
}
/// Set frame processor.
/// Frame processor.
public void SetFrameProcessor(IFrameProcessor processor)
{
NRDebugger.Info("[NRRender] SetFrameProcessor");
m_FrameProcessor = processor;
}
/// Pause render.
public void Pause()
{
NRDebugger.Info("[NRRender] Pause");
if (m_CurrentState != RendererState.Running)
{
return;
}
frameReady = false;
GL.IssuePluginEvent(RenderThreadHandlePtr, PAUSENATIVERENDEREVENT);
}
/// Resume render.
public void Resume()
{
Invoke("DelayResume", 0.3f);
}
/// Delay resume.
private void DelayResume()
{
NRDebugger.Info("[NRRender] Resume");
if (m_CurrentState != RendererState.Paused)
{
return;
}
GL.IssuePluginEvent(RenderThreadHandlePtr, RESUMENATIVERENDEREVENT);
}
#if !UNITY_EDITOR
void Update()
{
if (m_CurrentState == RendererState.Running)
{
// NRDebugger.Info("[NRRender] Update: frameCnt={0}", Time.frameCount);
if (m_FrameProcessor == null)
{
leftCamera.targetTexture = eyeTextures[currentEyeTextureIdx];
rightCamera.targetTexture = eyeTextures[currentEyeTextureIdx + 1];
currentEyeTextureIdx = nextEyeTextureIdx;
nextEyeTextureIdx = (nextEyeTextureIdx + 2) % EyeTextureCount;
}
else
{
m_FrameProcessor.Update();
}
leftCamera.enabled = true;
rightCamera.enabled = true;
frameReady = true;
}
else
{
frameReady = false;
}
}
#endif
/// Generates a render texture.
/// The width.
/// The height.
/// The render texture.
private RenderTexture GenRenderTexture(int width, int height)
{
return UnityExtendedUtility.CreateRenderTexture((int)(width * ScaleFactor), (int)(height * ScaleFactor), 24, RenderTextureFormat.Default);
}
/// Creates render textures.
private void CreateRenderTextures()
{
var config = NRSessionManager.Instance.NRSessionBehaviour?.SessionConfig;
if (config != null && config.UseMultiThread)
{
_TextureBufferSize = 5;
}
else
{
_TextureBufferSize = 4;
}
NRDebugger.Info("[NRRender] Texture buffer size:{0}", _TextureBufferSize);
EyeTextureCount = _TextureBufferSize * (int)Eyes.Count;
eyeTextures = new RenderTexture[EyeTextureCount];
var resolution = NRDevice.Subsystem.GetDeviceResolution(NativeDevice.LEFT_DISPLAY);
NRDebugger.Info("[CreateRenderTextures] Resolution :" + resolution.ToString());
for (int i = 0; i < EyeTextureCount; i++)
{
eyeTextures[i] = GenRenderTexture(resolution.width, resolution.height);
m_RTDict.Add(eyeTextures[i], eyeTextures[i].GetNativeTexturePtr());
}
}
/// Renders the coroutine.
/// An IEnumerator.
private IEnumerator RenderCoroutine()
{
WaitForEndOfFrame delay = new WaitForEndOfFrame();
yield return delay;
while (true)
{
yield return delay;
// NRDebugger.Info("[NRRender] RenderCoroutine: state={0}, isTrackingChanging={1}, frameProcessor={2}", m_CurrentState, m_IsTrackChanging, m_FrameProcessor != null);
if (m_CurrentState != RendererState.Running || !frameReady)
{
continue;
}
NativeMat4f apiPose;
Pose unityPose = NRFrame.HeadPose;
ConversionUtility.UnityPoseToApiPose(unityPose, out apiPose);
// NRDebugger.Info("[NRRender] unityPos={0}\napiPos={1}", unityPose.ToString("F6"), apiPose.ToString());
FrameInfo info = new FrameInfo(IntPtr.Zero, IntPtr.Zero, apiPose, new Vector3(0, 0, -m_FocusDistance),
Vector3.forward, NRFrame.CurrentPoseTimeStamp, m_FrameChangedType, NRTextureType.NR_TEXTURE_2D, 0);
if (m_FrameProcessor == null)
{
if (m_IsTrackChanging)
{
info.leftTex = m_HijackRenderTextureLeft.GetNativeTexturePtr();
info.rightTex = m_HijackRenderTextureRight.GetNativeTexturePtr();
}
else
{
IntPtr left_target, right_target;
if (!m_RTDict.TryGetValue(leftCamera.targetTexture, out left_target)) continue;
if (!m_RTDict.TryGetValue(rightCamera.targetTexture, out right_target)) continue;
info.leftTex = left_target;
info.rightTex = right_target;
}
UInt64 frame_handle = NativeRenderring.CreateFrameHandle();
info.frameHandle = frame_handle;
// NRDebugger.Info("[NRRender] RenderCoroutine: frameHandle={0}", frame_handle);
NativeRenderring?.WriteFrameData(info, 0, true);
GL.IssuePluginEvent(RenderThreadHandlePtr, SETRENDERTEXTUREEVENT);
}
else
{
info.frameHandle = m_FrameProcessor.GetFrameHandle();
var viewportListHandle = m_FrameProcessor.GetViewPortListHandle();
// NRDebugger.Info("[NRRender] RenderCoroutine FrameProcessor: frameCnt={0}, frameHandle={1}, viewportListHandle={2}", Time.frameCount, info.frameHandle, viewportListHandle);
NativeRenderring?.WriteFrameData(info, viewportListHandle, false);
GL.IssuePluginEvent(RenderThreadHandlePtr, SUBMIT_EVENT);
}
// reset focuse distance and frame changed type to default value every frame.
// m_FocusDistance = m_DefaultFocusDistance;
// m_FrameChangedType = NRFrameFlags.NR_FRAME_CHANGED_NONE;
}
}
private NRFrameFlags m_FrameChangedType = NRFrameFlags.NR_FRAME_CHANGED_NONE;
/// Sets the focus plane for render thread.
/// The distance from plane to center camera.
public void SetFocusDistance(float distance)
{
m_FocusDistance = distance;
m_FrameChangedType = NRFrameFlags.NR_FRAME_CHANGED_FOCUS_PLANE;
}
/// Executes the 'on render thread' operation.
/// Identifier for the event.
[MonoPInvokeCallback(typeof(RenderEventDelegate))]
private static void RunOnRenderThread(int eventID)
{
if (eventID != SETRENDERTEXTUREEVENT && eventID != SUBMIT_EVENT)
NRDebugger.Info("[NRRender] RunOnRenderThread : eventID={0}, frameCnt={1}", eventID, Time.frameCount);
if (eventID == STARTNATIVERENDEREVENT)
{
NativeRenderring?.Start();
var renderer = NRSessionManager.Instance.NRRenderer;
renderer.CurrentState = RendererState.Running;
if (renderer.m_FrameProcessor != null)
renderer.m_FrameProcessor.Initialize(NativeRenderring);
}
else if (eventID == RESUMENATIVERENDEREVENT)
{
NativeRenderring?.Resume();
NRSessionManager.Instance.NRRenderer.CurrentState = RendererState.Running;
}
else if (eventID == PAUSENATIVERENDEREVENT)
{
NRSessionManager.Instance.NRRenderer.CurrentState = RendererState.Paused;
NativeRenderring?.Pause();
}
// else if (eventID == STOPNATIVERENDEREVENT)
// {
// NativeRenderring?.Destroy();
// NativeRenderring = null;
// NRDevice.Instance.Destroy();
// }
else if (eventID == SETRENDERTEXTUREEVENT)
{
NativeRenderring?.DoExtendedRenderring();
}
else if (eventID == SUBMIT_EVENT)
{
var renderer = NRSessionManager.Instance.NRRenderer;
NativeRenderring?.DoSubmitFrame(renderer.m_FrameProcessor);
}
}
public void Destroy()
{
if (m_CurrentState == RendererState.Destroyed)
{
return;
}
m_CurrentState = RendererState.Destroyed;
//GL.IssuePluginEvent(RenderThreadHandlePtr, STOPNATIVERENDEREVENT);
if (m_FrameProcessor != null)
m_FrameProcessor.Destroy();
NativeRenderring?.Destroy();
NativeRenderring = null;
}
private void OnDestroy()
{
this.Destroy();
}
}
}