/**************************************************************************** * Copyright 2019 Nreal Techonology Limited. All rights reserved. * * This file is part of NRSDK. * * https://www.nreal.ai/ * *****************************************************************************/ namespace NRKernal { using System; using System.Collections.Generic; using UnityEngine; /// A manager of hand states. public class NRHandsManager { private readonly Dictionary m_HandsDict; private readonly HandState[] m_HandStates; // index 0 represents right and index 1 represents left private readonly OneEuroFilter[] m_OneEuroFilters; private IHandStatesService m_HandStatesService; private bool m_Inited; public Action OnHandTrackingStarted; public Action OnHandStatesUpdated; public Action OnHandTrackingStopped; /// /// Returns true if the hand tracking is now running normally /// public bool IsRunning { get { if (m_HandStatesService != null) { return m_HandStatesService.IsRunning; } return false; } } public NRHandsManager() { m_HandsDict = new Dictionary(); m_HandStates = new HandState[2] { new HandState(HandEnum.RightHand), new HandState(HandEnum.LeftHand) }; m_OneEuroFilters = new OneEuroFilter[2] { new OneEuroFilter(), new OneEuroFilter() }; } /// /// Regist the left or right NRHand. There would be at most one NRHand for each hand enum /// /// internal void RegistHand(NRHand hand) { if (hand == null || hand.HandEnum == HandEnum.None) return; var handEnum = hand.HandEnum; if (m_HandsDict.ContainsKey(handEnum)) { m_HandsDict[handEnum] = hand; } else { m_HandsDict.Add(handEnum, hand); } } /// /// UnRegist the left or right NRHand /// /// internal void UnRegistHand(NRHand hand) { if (hand == null) return; m_HandsDict.Remove(hand.HandEnum); } /// /// Init hand tracking with a certain service /// /// internal void Init(IHandStatesService handStatesService = null) { if (m_Inited) return; m_HandStatesService = handStatesService; if (m_HandStatesService == null) { #if UNITY_EDITOR m_HandStatesService = new NREmulatorHandStatesService(); #else m_HandStatesService = new NRHandStatesService(); #endif } NRInput.OnControllerStatesUpdated += UpdateHandTracking; m_Inited = true; NRDebugger.Info("[HandsManager] Hand Tracking Inited"); } /// /// Returns true if start hand tracking success /// /// internal bool StartHandTracking() { if (!m_Inited) { Init(); } if (IsRunning) { NRDebugger.Info("[HandsManager] Hand Tracking Start: Success"); return true; } if (m_HandStatesService != null && m_HandStatesService.RunService()) { NRDebugger.Info("[HandsManager] Hand Tracking Start: Success"); NRInput.SwitchControllerProvider(typeof(NRHandControllerProvider)); OnHandTrackingStarted?.Invoke(); return true; } NRDebugger.Info("[HandsManager] Hand Tracking Start: Failed"); return false; } /// /// Returns true if stop hand tracking success /// /// internal bool StopHandTracking() { if (!m_Inited) { NRDebugger.Info("[HandsManager] Hand Tracking Stop: Success"); return true; } if (!IsRunning) { NRDebugger.Info("[HandsManager] Hand Tracking Stop: Success"); return true; } if (m_HandStatesService != null && m_HandStatesService.StopService()) { NRDebugger.Info("[HandsManager] Hand Tracking Stop: Success"); NRInput.SwitchControllerProvider(ControllerProviderFactory.controllerProviderType); ResetHandStates(); OnHandTrackingStopped?.Invoke(); return true; } NRDebugger.Info("[HandsManager] Hand Tracking Stop: Failed"); return false; } /// /// Get the current hand state of the left or right hand /// /// /// public HandState GetHandState(HandEnum handEnum) { switch (handEnum) { case HandEnum.RightHand: return m_HandStates[0]; case HandEnum.LeftHand: return m_HandStates[1]; default: break; } return null; } /// /// Get the left or right NRHand if it has been registed. /// /// /// public NRHand GetHand(HandEnum handEnum) { NRHand hand; if (m_HandsDict != null && m_HandsDict.TryGetValue(handEnum, out hand)) { return hand; } return null; } /// /// Returns true if user is now performing the systemGesture /// /// public bool IsPerformingSystemGesture() { return IsPerformingSystemGesture(HandEnum.LeftHand) || IsPerformingSystemGesture(HandEnum.RightHand); } /// /// Returns true if user is now performing the systemGesture /// /// /// public bool IsPerformingSystemGesture(HandEnum handEnum) { return IsPerformingSystemGesture(GetHandState(handEnum)); } private void ResetHandStates() { for (int i = 0; i < m_HandStates.Length; i++) { m_HandStates[i].Reset(); } } private void UpdateHandTracking() { if (!IsRunning) return; m_HandStatesService.UpdateStates(m_HandStates); UpdateHandPointer(); OnHandStatesUpdated?.Invoke(); } private void UpdateHandPointer() { for (int i = 0; i < m_HandStates.Length; i++) { var handState = m_HandStates[i]; if (handState == null) continue; CalculatePointerPose(handState); } } private void CalculatePointerPose(HandState handState) { if (handState.isTracked) { var wristPose = handState.GetJointPose(HandJointID.Wrist); var cameraTransform = NRInput.CameraCenter; handState.pointerPoseValid = Vector3.Angle(cameraTransform.forward, wristPose.forward) < 70f; if (handState.pointerPoseValid) { Vector3 middleToRing = (handState.GetJointPose(HandJointID.MiddleProximal).position - handState.GetJointPose(HandJointID.RingProximal).position).normalized; Vector3 middleToWrist = (handState.GetJointPose(HandJointID.MiddleProximal).position - handState.GetJointPose(HandJointID.Wrist).position).normalized; Vector3 middleToCenter = Vector3.Cross(middleToWrist, middleToRing).normalized; var pointerPosition = handState.GetJointPose(HandJointID.MiddleProximal).position + middleToWrist * 0.02f + middleToRing * 0.01f + middleToCenter * (handState.handEnum == HandEnum.RightHand ? 0.06f : -0.06f); var handDirection = pointerPosition - wristPose.position; var handRotation = Quaternion.LookRotation(handDirection); var handtoCamera = wristPose.position - (cameraTransform.position - 0.08f * cameraTransform.forward); float handtoCameraY = handtoCamera.y; handtoCamera.y = 0; var handtoCameraRot = Quaternion.LookRotation(handtoCamera); var pointerRotation = Quaternion.Lerp(handtoCameraRot, handRotation, 0.3f) * Quaternion.Euler(new Vector3(handtoCameraY * -150f - 30f, handState.handEnum == HandEnum.RightHand ? -15f : 15f, 0f)); Vector3 pointerRotationToV3 = pointerRotation * Vector3.forward; pointerRotation = Quaternion.LookRotation(m_OneEuroFilters[(int)handState.handEnum].Step(Time.realtimeSinceStartup, pointerRotationToV3)); handState.pointerPose = new Pose(pointerPosition, pointerRotation); } } else { handState.pointerPoseValid = false; } } private bool IsPerformingSystemGesture(HandState handState) { if (!IsRunning) { return false; } return handState.currentGesture == HandGesture.System; } public class OneEuroFilter { public float Beta = 0.01f; public float MinCutoff = 2.0f; const float DCutOff = 1.0f; (float t, Vector3 x, Vector3 dx) _prev; public Vector3 Step(float t, Vector3 x) { var t_e = t - _prev.t; if (t_e < 1e-5f) return _prev.x; var dx = (x - _prev.x) / t_e; var dx_res = Vector3.Lerp(_prev.dx, dx, Alpha(t_e, DCutOff)); var cutoff = MinCutoff + Beta * dx_res.magnitude; var x_res = Vector3.Lerp(_prev.x, x, Alpha(t_e, cutoff)); _prev = (t, x_res, dx_res); return x_res; } static float Alpha(float t_e, float cutoff) { var r = 2 * Mathf.PI * cutoff * t_e; return r / (r + 1); } } } }