using System.Text; using System; using UnityEngine; using UnityEngine.EventSystems; using Rokid.UXR.Module; using Rokid.UXR.Utility; namespace Rokid.UXR.Interaction { #region Data /// /// Types of Gesture Interaction /// public enum InteractorType { None, //near-field interaction Near, //far-field interaction Far } /// /// Hand bone node marker index /// public enum SkeletonIndexFlag { WRIST = 0, THUMB_CMC = 1, THUMB_MCP = 2, THUMB_IP = 3, THUMB_TIP = 4, INDEX_FINGER_MCP = 5, INDEX_FINGER_PIP = 6, INDEX_FINGER_DIP = 7, INDEX_FINGER_TIP = 8, MIDDLE_FINGER_MCP = 9, MIDDLE_FINGER_PIP = 10, MIDDLE_FINGER_DIP = 11, MIDDLE_FINGER_TIP = 12, RING_FINGER_MCP = 13, RING_FINGER_PIP = 14, RING_FINGER_DIP = 15, RING_FINGER_TIP = 16, PINKY_MCP = 17, PINKY_PIP = 18, PINKY_DIP = 19, PINKY_TIP = 20 } /// /// Gesture data class /// [Serializable] public class Gesture { /// /// Types of hands /// public HandType handType; /// /// Types of gestures /// public GestureType gesType; /// /// Pressed in a certain frame /// public bool handDown; /// /// Released in a certain frame /// public bool handUp; /// /// Hand is Click /// public bool handClick; /// /// Hand is Pressing /// public bool handPress; /// /// Hand is tracking succcess /// public bool trackingSuccess; /// /// Hand Center Position /// public Vector3 position; /// /// Hand delta position /// public Vector3 deltaPos; /// /// Hand orientation /// public HandOrientation handOrientation; public void Reset() { gesType = GestureType.None; handDown = false; handUp = false; handClick = false; handPress = false; } public Gesture(HandType type) { this.handType = type; } public override string ToString() { StringBuilder builder = new StringBuilder(); builder.Append("HandType:").Append(handType.ToString()) .Append(" GesType:").Append(gesType.ToString()) .Append(" HandDown:").Append(handDown) .Append(" HandUp:").Append(handUp) .Append(" HandClick:").Append(handClick) .Append(" HandPress:").Append(handPress) .Append(" HandOrientation:").Append(handOrientation) .Append(" Position:").Append(position) .Append(" deltaPos:").Append(deltaPos); return builder.ToString(); } } /// /// Types of gestures /// public enum GestureType { None = -1, Grip = 1, Palm = 2, Pinch = 3, OpenPinch = 4, } /// /// Types of hands /// public enum HandType { None = 0, LeftHand = 2, RightHand = 1 } /// /// Types of handoientation /// public enum HandOrientation { None = -1, Palm = 0, Back = 1 } /// /// NormalHand / HeadHand /// public enum HandOrHeadHandType { NormalHand, HeadHand } #endregion /// /// Gesture event input, provide all gesture external interface /// public class GesEventInput : MonoSingleton, IEventInput { #region Event /// /// Hand click callback /// /// Hand type public static Action OnGesClick; /// /// hand press callback /// /// Hand type public static Action OnHandPress; /// /// Hand release callback /// /// Hand type public static Action OnHandRelease; /// /// Gesture tracked failed callback /// /// handtype=leftHand lefthand lost, handtype=rightHand righthand lost public static Action OnTrackedFailed; /// /// Hand track success callback update invoke /// /// hand type public static Action OnTrackedSuccess; /// /// Process Gesture Data /// /// The result of the gesture data returned public static Action OnProcessGesData; /// /// Use log ges fps /// /// wrist pos public static Action OnGesDataUpdate; /// /// Use to proceess ges log /// /// The time difference between the upper and lower frames of the gesture public static Action OnLogGesData; /// /// Use to process ray pose /// /// Hand type lefthand/righthand /// Wrist pos /// Handcenter pos /// Pinchcenter base pos /// Pinchcenter pos public static Action OnRayPoseUpdate; /// /// Use to render hand update invoke /// /// hand type public static Action OnRenderHand; /// /// On handlost in camera space callback /// /// hand type public static Action OnHandLostInCameraSpace; /// /// On hand orientaion change callback update invoke /// public static Action OnHandOrientationUpdate; /// /// Gesture module action callback /// public static Action OnActiveGesModule; /// /// Gesture module release callback /// public static Action OnReleaseGesModule; /// /// Gesture module init callback /// public static Action OnInitialzeGesModule; /// /// Hand or headhand type change callback /// public static Action OnHandOrHeadHandTypeChange; #endregion [SerializeField] private GesImplementation gesInput; /// /// Interator /// /// public Transform Interactor { get; set; } /// /// Auto change headhand or handtype /// /// public bool autoChangeHeadHandOrHandType = false; private bool initialize = false; private GesImplementation GesInput { get { if (gesInput == null) { if (gameObject.GetComponent() != null) { gesInput = gameObject.GetComponent(); } else { gesInput = gameObject.AddComponent(); } } return gesInput; } } /// /// Gesture Module Initialize /// /// Generates the parent of the interactor public void Initialize(Transform parent) { if (Utils.IsAndroidPlatfrom() && !FuncDeviceCheck.CheckHandTrackingFunc()) { return; } #if UNITY_EDITOR if (GetComponent() == null) gameObject.AddComponent(); #endif if (Interactor == null) { GameObject go = GameObject.Find("RKHand"); if (go == null) { go = GameObject.Instantiate(Resources.Load("Prefabs/Interactor/RKHand")); } go.name = "RKHand"; Interactor = go.transform; Interactor.SetParent(transform); } Interactor.SetParent(transform); if (parent != null) this.transform.SetParent(parent); OnInitialzeGesModule?.Invoke(); ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand); GesInput.Initialze(); initialize = true; RKLog.KeyInfo("====GesEventInit==== Init"); } /// /// Activate hand interaction or head-hand interaction /// /// public void ActiveHandOrHeadHand(HandOrHeadHandType handOrHeadHandType) { OnHandOrHeadHandTypeChange?.Invoke(handOrHeadHandType); } /// /// Release Gesture /// public void Release() { OnReleaseGesModule?.Invoke(); Destroy(this.gameObject); } /// /// OnDestroy /// protected override void OnDestroy() { OnGesClick = null; OnTrackedFailed = null; OnHandPress = null; OnHandRelease = null; OnTrackedSuccess = null; OnProcessGesData = null; OnGesDataUpdate = null; OnLogGesData = null; OnRenderHand = null; initialize = false; } /// /// Active Gesture /// public void ActiveModule() { RKLog.KeyInfo("====GesEventInput==== : ActiveModule"); OnActiveGesModule?.Invoke(); RKVirtualController.Instance.Change(ControllerType.NORMAL); RKVirtualController.Instance.UseCustomGamePadEvent(false); EventSystem.current.pixelDragThreshold = 60; } /// /// Get Hand Pose /// /// handtype lefthand righthand /// public Vector3 GetHandDeltaPos(HandType hand) { if (!initialize) { return Vector3.zero; }; return GesInput.GetHandDeltaPos(hand); } /// /// GetHandDown /// /// left/right hand /// true pinch / false grip /// public bool GetHandDown(HandType type, bool isPinch) { if (!initialize) { return false; }; return GesInput.GetHandDown(type, isPinch); } /// /// Get Hand Up /// /// left/right hand /// true pinch / false grip /// When hand up return true or return false public bool GetHandUp(HandType type, bool isPinch) { if (!initialize) { return false; }; return GesInput.GetHandUp(type, isPinch); } /// /// Get Hand Press /// /// left/right hand /// true pinch / false grip /// When hand press return true or return false public bool GetHandPress(HandType type, bool isPinch) { if (!initialize) { return false; }; return GesInput.GetHandPress(type, isPinch); } /// /// Get hand down /// /// left/right hand /// public bool GetHandDown(HandType type) { if (!initialize) { return false; }; return GesInput.GetHandDown(type); } /// /// Get hand press /// /// left/right hand /// public bool GetHandPress(HandType type) { if (!initialize) { return false; }; return GesInput.GetHandPress(type); } /// /// Set user height default 170 cm /// /// public void SetUserHeight(float height) { if (!initialize) { return; }; gesInput.SetUserHeight(height); } /// /// Get hand up /// /// left/right hand /// public bool GetHandUp(HandType type) { if (!initialize) { return false; }; return GesInput.GetHandUp(type); } /// /// Get current gesture info /// /// left/right hand /// public Gesture GetGesture(HandType type) { if (!initialize) { return default(Gesture); }; return GesInput.GetGesture(type); } /// /// Get current gesture type /// /// left/right hand /// public GestureType GetGestureType(HandType type) { if (!initialize) { return GestureType.None; }; return GesInput.GetGestureType(type); } /// /// Get current hand center position /// /// left/right hand /// public Vector3 GetHandPos(HandType handType) { if (!initialize) { return Vector3.zero; }; return GesInput.GetHandPos(handType); } /// /// Get current hand pose /// /// left/right hand /// hand pose public Pose GetHandPose(HandType handType) { if (!initialize) { return Pose.identity; }; return GesInput.GetHandPose(handType); } /// /// Get Skeleton Pose /// /// Seleton index flag /// left/right hand /// public Pose GetSkeletonPose(SkeletonIndexFlag flag, HandType type) { if (!initialize) { return Pose.identity; }; return GesInput.GetSkeletonPose(flag, type); } /// /// Get hand interactor type /// /// left/right hand /// InteractorType Near/Far public InteractorType GetInteractorType(HandType hand) { if (!initialize) { return InteractorType.None; }; return GesInput.GetInteractorType(hand); } /// /// Set interacor type /// /// InteractorType Near/Far /// left/right hand public void SetInteractorType(InteractorType type, HandType hand) { if (!initialize) { return; }; GesInput.SetInteractorType(type, hand); } /// /// Get hand orientation /// /// left/right hand /// public HandOrientation GetHandOrientation(HandType hand) { if (!initialize) { return HandOrientation.None; }; return GesInput.GetHandOrientation(hand); } /// /// Unity Update /// private void Update() { if (!initialize) return; #if UNITY_EDITOR if (Input.GetKeyDown(KeyCode.G) || Input.GetKeyDown(KeyCode.F)) { this.ActiveModule(); } if (Input.GetKeyDown(KeyCode.O)) { ActiveHandOrHeadHand(HandOrHeadHandType.HeadHand); } if (Input.GetKeyDown(KeyCode.I)) { ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand); } #else if (GetHandDown(HandType.LeftHand) && GetHandOrientation(HandType.LeftHand) == HandOrientation.Palm) { ProcessActiveModule(HandType.LeftHand); if (autoChangeHeadHandOrHandType) ActiveHandOrHeadHand(HandOrHeadHandType.HeadHand); } if (GetHandDown(HandType.RightHand) && GetHandOrientation(HandType.RightHand) == HandOrientation.Palm) { ProcessActiveModule(HandType.RightHand); if (autoChangeHeadHandOrHandType) ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand); } #endif } /// /// Process module active logic /// /// left/right hand private void ProcessActiveModule(HandType hand) { Vector3 handPos = GesEventInput.Instance.GetHandPose(hand).position; Vector3 camereSpaceHandPos = MainCameraCache.mainCamera.transform.InverseTransformPoint(handPos); float hFov = Vector3.Angle(Vector3.forward, new Vector3(camereSpaceHandPos.x, 0, camereSpaceHandPos.z)); float vFov = Vector3.Angle(Vector3.forward, new Vector3(0, camereSpaceHandPos.y, camereSpaceHandPos.z)); // RKLog.Debug($"====GesEventInput==== hFov:{hFov}, vFov:{vFov}"); if (hFov < 30 && vFov < 20) { this.ActiveModule(); } } } }