using System.Collections; using System.Collections.Generic; using UnityEngine; using System; namespace Ximmerse.XR.InputSystems.GazeAndGestureInteraction { /// /// 更新 tracking state 的状态控制器 /// public partial class GazeAndHandInteractionSystem { /// /// Gesture type recoginzed by system /// public enum GestureType : byte { /// /// 不明手势 /// Unkown = 0, /// /// 手指张开并朝向用户前方 /// OpenHandAndPointForward, /// /// 手指张开并朝向用户自身方向 /// OpenHandAndPointToUser, /// /// 食指拇指捏合 /// Pinch, /// /// 握拳并手心朝向用户前方 /// CloseHandAndPointForward, /// /// 握拳并手心朝向用户自身 /// CloseHandAndPointToUser, } /// /// Impl of gesture tracking state. /// internal partial class TrackState { Camera m_mainCamera; public Camera mainCamera { get { if (!m_mainCamera) m_mainCamera = Camera.main; return m_mainCamera; } } /// /// Current gesture type /// public GestureType gestureType { get; private set; } /// /// Delta position movement in world space /// public Vector3 DeltaPositionWorld { get => this.gestureRecords[0].PalmWorldPose.position - gestureRecords[1].PalmWorldPose.position; } /// /// Delta angular rotation in world space /// public Vector3 DeltaAngluarWorld { get => this.gestureRecords[0].PalmWorldPose.rotation.eulerAngles - gestureRecords[1].PalmWorldPose.rotation.eulerAngles; } public event Action OnGestureChanged; float previousGestureChangeTime; /// /// Structure to cache native gesture data. /// struct NativeGestureRecord { public bool IsValidFrame; public float RealFrameTime; /// /// Palm world pose /// public Pose PalmWorldPose; /// /// Palm local pose. /// public Pose PalmLocalPose; /// /// Palm movement velocity. /// public Pose PalmVelocity; /// /// distance to previous frame /// public float PalmDeltaDistance; /// /// Angle diff to previous frame /// public Vector3 DeltaAngle; public int NativeGesturePluginCode; } const int kCacheGestureDataCount = 5; /// /// 缓存的手势输入数据. /// readonly NativeGestureRecord[] gestureRecords = new NativeGestureRecord[kCacheGestureDataCount]; void TickGestureState() { UpdateGestureRecordQueue(); //连续三帧判断状态: if (CheckValidFrameCount(true, 2)) { trackingState = HandTrackingStatus.Tracking; } else if (CheckValidFrameCount(false, 2)) { trackingState = HandTrackingStatus.Invalid; } //don't change gesture type too frequently if (trackingState == HandTrackingStatus.Tracking && (Time.realtimeSinceStartup - previousGestureChangeTime) >= kMinAllowStateChangeTime) { UpdateGestureTypeChange(); } } /// /// 更新 gesture输入队列 /// void UpdateGestureRecordQueue() { HandTrackingInfo currentHandTrackInfo = HandTracking.HandTrackingInfo; if (!currentHandTrackInfo.IsValid) { InsertRecord(new NativeGestureRecord());//insert dummy record } //添加一条有效数据 else { NativeGestureRecord record = new NativeGestureRecord() { IsValidFrame = true, RealFrameTime = Time.realtimeSinceStartup, NativeGesturePluginCode = currentHandTrackInfo.NativeGestureType, PalmWorldPose = new Pose(currentHandTrackInfo.PalmPosition, Quaternion.LookRotation(currentHandTrackInfo.PalmNormal)), PalmLocalPose = new Pose(currentHandTrackInfo.PalmLocalPosition, Quaternion.LookRotation(currentHandTrackInfo.PalmLocalNormal)), }; //计算delta: if (GetPreviousValidFrame(out NativeGestureRecord prevValidFrame)) { float deltaTime = Mathf.Min(Mathf.Epsilon, record.RealFrameTime - prevValidFrame.RealFrameTime); Vector3 vector2prev = record.PalmWorldPose.position - prevValidFrame.PalmWorldPose.position; Vector3 deltaAngle = (Quaternion.Inverse(prevValidFrame.PalmWorldPose.rotation) * record.PalmWorldPose.rotation).eulerAngles; record.PalmVelocity = new Pose( (vector2prev) / deltaTime, Quaternion.Euler(deltaAngle / deltaTime)); record.PalmDeltaDistance = vector2prev.magnitude; record.DeltaAngle = deltaAngle; } InsertRecord(record); } } /// /// 更新手势识别的当前手势输入类型 /// void UpdateGestureTypeChange() { int checkFrame = 2; //如果连续五帧是Open Hand姿态: if (CheckNativeGestureCodeMatchContinueFrames((int)TouchlessA3D.GestureType.OPEN_HAND, checkFrame) || CheckNativeGestureCodeMatchContinueFrames((int)TouchlessA3D.GestureType.HAND, checkFrame)) { //手心张开,向前 if (CheckForawrdMatchContinueFrames(this.mainCamera.transform.forward, checkFrame)) { InternalUpdateGestureState(GestureType.OpenHandAndPointForward); } //手心张开,向后 else if (CheckForawrdMatchContinueFrames(-this.mainCamera.transform.forward, checkFrame)) { InternalUpdateGestureState(GestureType.OpenHandAndPointToUser); } else { InternalUpdateGestureState(GestureType.Unkown); } } if (CheckNativeGestureCodeMatchContinueFrames((int)TouchlessA3D.GestureType.CLOSED_PINCH, checkFrame)) { InternalUpdateGestureState(GestureType.Pinch); } if (CheckNativeGestureCodeMatchContinueFrames((int)TouchlessA3D.GestureType.CLOSED_HAND, checkFrame)) { //握拳,向前 if (CheckForawrdMatchContinueFrames(this.mainCamera.transform.forward, checkFrame)) { InternalUpdateGestureState(GestureType.CloseHandAndPointForward); } //握拳,向用户 else if (CheckForawrdMatchContinueFrames(-this.mainCamera.transform.forward, checkFrame)) { InternalUpdateGestureState(GestureType.CloseHandAndPointToUser); } else { InternalUpdateGestureState(GestureType.Unkown); } } } void InternalUpdateGestureState(GestureType newGesture) { if (gestureType != newGesture) { gestureType = newGesture; previousGestureChangeTime = Time.realtimeSinceStartup; OnGestureChanged?.Invoke(newGesture); } } void InsertRecord(NativeGestureRecord nativeRecord) { for (int i = gestureRecords.Length - 1; i > 0; i--) { gestureRecords[i] = gestureRecords[i - 1]; } gestureRecords[0] = nativeRecord; } /// /// Get previous valid record. /// /// /// bool GetPreviousValidFrame(out NativeGestureRecord prevNativeRecord) { for (int i = 0; i < gestureRecords.Length; i++)//starts from 1 { if (gestureRecords[i].IsValidFrame) { prevNativeRecord = gestureRecords[i]; return true; } } prevNativeRecord = default(NativeGestureRecord); return false; } /// /// 检查连续若干帧数据是有效数据 /// /// /// bool CheckValidFrameCount(bool isValid, int frameCount) { for (int i = 0; i < frameCount; i++) { if (gestureRecords[i].IsValidFrame != isValid) { return false; } } return true; } /// /// Check a certain frames matching forward direction /// /// /// /// /// bool CheckForawrdMatchContinueFrames(Vector3 forward, int frameCount, float expectDot = 0.45f) { for (int i = 0; i < frameCount; i++) { if (Vector3.Dot(forward, this.gestureRecords[i].PalmWorldPose.forward) < expectDot) { return false; } } return true; } /// /// Check if a certain frames matching target gesture code. /// /// /// /// bool CheckNativeGestureCodeMatchContinueFrames(int targetCode, int frameCount) { for (int i = 0; i < frameCount; i++) { if (targetCode != this.gestureRecords[i].NativeGesturePluginCode) { return false; } } return true; } } } }