/**************************************************************************** * Copyright 2019 Nreal Techonology Limited. All rights reserved. * * This file is part of NRSDK. * * https://www.nreal.ai/ * *****************************************************************************/ namespace NRKernal.Experimental { using System; using UnityEngine; [ExecuteInEditMode] public class NROverlay : OverlayBase { /// /// If true, the layer will be created as an external surface. externalSurfaceObject contains the Surface object. It's effective only on Android. /// [Tooltip("If true, the layer will be created as an external surface. externalSurfaceObject contains the Surface object. It's effective only on Android.")] public bool isExternalSurface = false; /// /// The width which will be used to create the external surface. It's effective only on Android. /// [Tooltip("The width which will be used to create the external surface. It's effective only on Android.")] public int externalSurfaceWidth = 0; /// /// The height which will be used to create the external surface. It's effective only on Android. /// [Tooltip("The height which will be used to create the external surface. It's effective only on Android.")] public int externalSurfaceHeight = 0; /// /// If true, the texture's content is copied to the compositor each frame. /// [Tooltip("If true, the texture's content is copied to the compositor each frame.")] public bool isDynamic = false; /// /// If true, the layer would be used to present protected content. The flag is effective only on Android. /// [Tooltip("If true, the layer would be used to present protected content. The flag is effective only on Android.")] public bool isProtectedContent = false; /// /// The Texture to show in the layer. /// [Tooltip("The Texture to show in the layer.")] public Texture texture; /// /// Which display this overlay should render to. /// [Tooltip("Which display this overlay should render to.")] public LayerSide layerSide = LayerSide.Both; /// /// Whether render this overlay as 0-dof. /// [Tooltip("Whether render this overlay as 0-dof.")] public bool isScreenSpace = false; /// /// Whether this overlay is 3D rendering layer. /// [Tooltip("Whether this overlay is 3D rendering layer.")] public bool is3DLayer = false; /// /// Preview the overlay in the editor using a mesh renderer. /// [Tooltip("Preview the overlay in the editor using a mesh renderer.")] public bool previewInEditor; private bool userTextureMask = false; public delegate void ExternalSurfaceObjectCreated(); public delegate void BufferedSurfaceObjectChangeded(RenderTexture rt); public event ExternalSurfaceObjectCreated externalSurfaceObjectCreated; /// Only for RenderTexture imageType. public event BufferedSurfaceObjectChangeded onBufferChanged; public Texture MainTexture { get { return texture; } set { if (texture != value) { SetDirty(true); userTextureMask = true; texture = value; } } } /// Determines the on-screen appearance of a layer. public enum OverlayShape { Quad, //Cylinder, //Cubemap, //OffcenterCubemap, //Equirect, } private NROverlayMeshGenerator m_MeshGenerator; public OverlayShape overlayShape { get; set; } = OverlayShape.Quad; private Matrix4x4 m_WorldToLeftDisplayMatrix = Matrix4x4.identity; private Matrix4x4 m_WorldToRightDisplayMatrix = Matrix4x4.identity; private Matrix4x4 m_OriginPose; public Rect[] sourceUVRect = new Rect[2] { new Rect(0, 0, 1, 1), new Rect(0, 0, 1, 1) }; new void Start() { base.Start(); ApplyMeshAndMat(); } private void ApplyMeshAndMat() { m_MeshGenerator = gameObject.GetComponentInChildren(); if (m_MeshGenerator == null) { var go = new GameObject("NROverlayMeshGenerator"); // go.hideFlags = HideFlags.HideInHierarchy; go.transform.SetParent(this.transform, false); m_MeshGenerator = go.AddComponent(); } m_MeshGenerator.SetOverlay(this); } protected override void Initialize() { base.Initialize(); m_BufferSpec = new BufferSpec(); if (isExternalSurface) { m_BufferSpec.size = new NativeResolution(externalSurfaceWidth, externalSurfaceHeight); } else if (texture != null) { m_BufferSpec.size = new NativeResolution(texture.width, texture.height); } m_BufferSpec.colorFormat = NRColorFormat.NR_COLOR_FORMAT_RGBA_8888; m_BufferSpec.depthFormat = NRDepthStencilFormat.NR_DEPTH_STENCIL_FORMAT_DEPTH_24; m_BufferSpec.samples = 1; m_BufferSpec.useExternalSurface = isExternalSurface; int flag = 0; if (isExternalSurface) { if (isProtectedContent) { flag |= (int)NRExternalSurfaceFlags.NR_EXTERNAL_SURFACE_FLAG_PROTECTED; } if (is3DLayer) { flag |= (int)NRExternalSurfaceFlags.NR_EXTERNAL_SURFACE_FLAG_SYNCHRONOUS; flag |= (int)NRExternalSurfaceFlags.NR_EXTERNAL_SURFACE_FLAG_USE_TIMESTAMPS; } } m_BufferSpec.surfaceFlag = flag; m_OriginPose = Matrix4x4.TRS(transform.position, transform.rotation, transform.lossyScale); } public override void CreateOverlayTextures() { ReleaseOverlayTextures(); IntPtr texturePtr = IntPtr.Zero; if (isExternalSurface) { externalSurfaceObjectCreated?.Invoke(); } else if (isDynamic) { if (texture == null) { NRDebugger.Warning("Current texture is empty!!!"); return; } for (int i = 0; i < m_BufferSpec.bufferCount; ++i) { RenderTexture rt = UnityExtendedUtility.CreateRenderTexture( texture.width, texture.height, 24, RenderTextureFormat.ARGB32 ); texturePtr = rt.GetNativeTexturePtr(); Textures.Add(texturePtr, rt); } } else if (texture) { texturePtr = texture.GetNativeTexturePtr(); Textures.Add(texturePtr, texture); } } public override void ReleaseOverlayTextures() { if (Textures.Count == 0) { return; } if (isDynamic) { foreach (var item in Textures) { RenderTexture rt = item.Value as RenderTexture; if (rt != null) { rt.Release(); } else if (item.Value != null) { GameObject.Destroy(item.Value); } } } Textures.Clear(); } public override void CreateViewport() { base.CreateViewport(); if (layerSide != LayerSide.Both) { m_ViewPorts = new ViewPort[1]; m_ViewPorts[0].reprojection = isScreenSpace ? NRReprojectionType.NR_REPROJECTION_TYPE_NONE : NRReprojectionType.NR_REPROJECTION_TYPE_FULL; m_ViewPorts[0].sourceUV = sourceUVRect[0]; m_ViewPorts[0].targetDisplay = layerSide; m_ViewPorts[0].layerID = m_LayerId; m_ViewPorts[0].is3DLayer = is3DLayer; if (is3DLayer) { NativeDevice displayDev = layerSide == LayerSide.Left ? NativeDevice.LEFT_DISPLAY : NativeDevice.RIGHT_DISPLAY; NRFrame.GetEyeFov(displayDev, ref m_ViewPorts[0].fov); } else { m_ViewPorts[0].transform = CalculateCoords(layerSide); } NRSwapChainManager.Instance.CreateBufferViewport(ref m_ViewPorts[0]); } else { m_ViewPorts = new ViewPort[2]; m_ViewPorts[0].reprojection = isScreenSpace ? NRReprojectionType.NR_REPROJECTION_TYPE_NONE : NRReprojectionType.NR_REPROJECTION_TYPE_FULL; m_ViewPorts[0].sourceUV = sourceUVRect[0]; m_ViewPorts[0].targetDisplay = LayerSide.Left; m_ViewPorts[0].layerID = m_LayerId; m_ViewPorts[0].index = -1; if (is3DLayer) { NRFrame.GetEyeFov(NativeDevice.LEFT_DISPLAY, ref m_ViewPorts[0].fov); } else { m_ViewPorts[0].transform = CalculateCoords(LayerSide.Left); } m_ViewPorts[0].is3DLayer = is3DLayer; NRSwapChainManager.Instance.CreateBufferViewport(ref m_ViewPorts[0]); m_ViewPorts[1].reprojection = isScreenSpace ? NRReprojectionType.NR_REPROJECTION_TYPE_NONE : NRReprojectionType.NR_REPROJECTION_TYPE_FULL; m_ViewPorts[1].sourceUV = sourceUVRect[1]; m_ViewPorts[1].targetDisplay = LayerSide.Right; m_ViewPorts[1].layerID = m_LayerId; m_ViewPorts[1].index = -1; if (is3DLayer) { NRFrame.GetEyeFov(NativeDevice.RIGHT_DISPLAY, ref m_ViewPorts[1].fov); } else { m_ViewPorts[1].transform = CalculateCoords(LayerSide.Right); } m_ViewPorts[1].is3DLayer = is3DLayer; NRSwapChainManager.Instance.CreateBufferViewport(ref m_ViewPorts[1]); } if (isScreenSpace) { CreateWorldToDisplayMatrix(); } if (is3DLayer) { ClearMask(); } } /// /// Update transform of viewPort. /// public override void UpdateViewPort() { base.UpdateViewPort(); if (m_ViewPorts == null) { NRDebugger.Warning("Can not update view port for this layer:{0}", gameObject.name); return; } if (layerSide != LayerSide.Both) { if (!is3DLayer) { m_ViewPorts[0].transform = CalculateCoords(layerSide); } NRSwapChainManager.Instance.UpdateBufferViewport(ref m_ViewPorts[0]); } else { if (!is3DLayer) { m_ViewPorts[0].transform = CalculateCoords(LayerSide.Left); m_ViewPorts[1].transform = CalculateCoords(LayerSide.Right); } NRSwapChainManager.Instance.UpdateBufferViewport(ref m_ViewPorts[0]); NRSwapChainManager.Instance.UpdateBufferViewport(ref m_ViewPorts[1]); } } public override void DestroyViewPort() { base.DestroyViewPort(); if (m_ViewPorts != null) { foreach (var viewport in m_ViewPorts) { NRSwapChainManager.Instance.DestroyBufferViewPort(viewport.nativeHandler); } m_ViewPorts = null; } } public void Apply() { SetDirty(true); } public override void SwapBuffers(IntPtr bufferHandler) { if (isDynamic) { Texture targetTexture; if (!Textures.TryGetValue(bufferHandler, out targetTexture)) { NRDebugger.Error("Can not find the texture:" + bufferHandler); return; } onBufferChanged?.Invoke((RenderTexture)targetTexture); } } public IntPtr GetSurfaceId() { if (!isExternalSurface) { return IntPtr.Zero; } return NRSwapChainManager.Instance.GetSurfaceHandler(this.LayerId); } private void CreateWorldToDisplayMatrix() { var centerAnchor = NRSessionManager.Instance.NRHMDPoseTracker.centerAnchor; Matrix4x4 head_T_centerLocal = Matrix4x4.TRS( centerAnchor.transform.localPosition, centerAnchor.transform.localRotation, centerAnchor.transform.localScale); var leftCamera = NRSessionManager.Instance.NRHMDPoseTracker.leftCamera; Matrix4x4 head_T_leftLocal = Matrix4x4.TRS( leftCamera.transform.localPosition, leftCamera.transform.localRotation, leftCamera.transform.localScale); m_WorldToLeftDisplayMatrix = leftCamera.worldToCameraMatrix * leftCamera.transform.localToWorldMatrix * Matrix4x4.Inverse(head_T_leftLocal) * head_T_centerLocal; var rightCamera = NRSessionManager.Instance.NRHMDPoseTracker.rightCamera; Matrix4x4 head_T_rightLocal = Matrix4x4.TRS( rightCamera.transform.localPosition, rightCamera.transform.localRotation, rightCamera.transform.localScale); m_WorldToRightDisplayMatrix = rightCamera.worldToCameraMatrix * rightCamera.transform.localToWorldMatrix * Matrix4x4.Inverse(head_T_rightLocal) * head_T_centerLocal; } public void UpdateTransform(Matrix4x4 originMatrix, bool isScreenSpace) { this.isScreenSpace = isScreenSpace; if (this.isScreenSpace) { CreateWorldToDisplayMatrix(); } m_OriginPose = originMatrix; } private Matrix4x4 CalculateCoords(LayerSide side) { if (isScreenSpace) { if (side == LayerSide.Left) { return m_WorldToLeftDisplayMatrix * m_OriginPose; } else { return m_WorldToRightDisplayMatrix * m_OriginPose; } } else { Camera imageCamera = null; if (side == LayerSide.Left) { imageCamera = NRSessionManager.Instance.NRHMDPoseTracker.leftCamera; } else if (side == LayerSide.Right) { imageCamera = NRSessionManager.Instance.NRHMDPoseTracker.rightCamera; } if (imageCamera == null) { return transform.localToWorldMatrix; } return imageCamera.worldToCameraMatrix * transform.localToWorldMatrix; } } public void UpdateExternalSurface(NativeMat4f pose, Int64 timestamp, int index) { NRSwapChainManager.Instance.UpdateExternalSurface(m_LayerId, pose, timestamp, index); } private void ClearMask() { if (m_MeshGenerator != null) { GameObject.DestroyImmediate(m_MeshGenerator.gameObject); m_MeshGenerator = null; } } public override void Destroy() { base.Destroy(); if (userTextureMask) { texture = null; userTextureMask = false; } } new void OnDestroy() { base.OnDestroy(); ClearMask(); } } }