123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628 |
- using System.Text;
- using System;
- using UnityEngine;
- using UnityEngine.EventSystems;
- using Rokid.UXR.Module;
- using Rokid.UXR.Utility;
- namespace Rokid.UXR.Interaction
- {
- #region Data
- /// <summary>
- /// Types of Gesture Interaction
- /// </summary>
- public enum InteractorType
- {
- None,
- //near-field interaction
- Near,
- //far-field interaction
- Far
- }
- /// <summary>
- /// Hand bone node marker index
- /// </summary>
- public enum SkeletonIndexFlag
- {
- WRIST = 0,
- THUMB_CMC = 1,
- THUMB_MCP = 2,
- THUMB_IP = 3,
- THUMB_TIP = 4,
- INDEX_FINGER_MCP = 5,
- INDEX_FINGER_PIP = 6,
- INDEX_FINGER_DIP = 7,
- INDEX_FINGER_TIP = 8,
- MIDDLE_FINGER_MCP = 9,
- MIDDLE_FINGER_PIP = 10,
- MIDDLE_FINGER_DIP = 11,
- MIDDLE_FINGER_TIP = 12,
- RING_FINGER_MCP = 13,
- RING_FINGER_PIP = 14,
- RING_FINGER_DIP = 15,
- RING_FINGER_TIP = 16,
- PINKY_MCP = 17,
- PINKY_PIP = 18,
- PINKY_DIP = 19,
- PINKY_TIP = 20
- }
- /// <summary>
- /// Gesture data class
- /// </summary>
- [Serializable]
- public class Gesture
- {
- /// <summary>
- /// Types of hands
- /// </summary>
- public HandType handType;
- /// <summary>
- /// Types of gestures
- /// </summary>
- public GestureType gesType;
- /// <summary>
- /// Pressed in a certain frame
- /// </summary>
- public bool handDown;
- /// <summary>
- /// Released in a certain frame
- /// </summary>
- public bool handUp;
- /// <summary>
- /// Hand is Click
- /// </summary>
- public bool handClick;
- /// <summary>
- /// Hand is Pressing
- /// </summary>
- public bool handPress;
- /// <summary>
- /// Hand is tracking succcess
- /// </summary>
- public bool trackingSuccess;
- /// <summary>
- /// Hand Center Position
- /// </summary>
- public Vector3 position;
- /// <summary>
- /// Hand delta position
- /// </summary>
- public Vector3 deltaPos;
- /// <summary>
- /// Hand orientation
- /// </summary>
- public HandOrientation handOrientation;
- public void Reset()
- {
- gesType = GestureType.None;
- handDown = false;
- handUp = false;
- handClick = false;
- handPress = false;
- }
- public Gesture(HandType type)
- {
- this.handType = type;
- }
- public override string ToString()
- {
- StringBuilder builder = new StringBuilder();
- builder.Append("HandType:").Append(handType.ToString())
- .Append(" GesType:").Append(gesType.ToString())
- .Append(" HandDown:").Append(handDown)
- .Append(" HandUp:").Append(handUp)
- .Append(" HandClick:").Append(handClick)
- .Append(" HandPress:").Append(handPress)
- .Append(" HandOrientation:").Append(handOrientation)
- .Append(" Position:").Append(position)
- .Append(" deltaPos:").Append(deltaPos);
- return builder.ToString();
- }
- }
- /// <summary>
- /// Types of gestures
- /// </summary>
- public enum GestureType
- {
- None = -1,
- Grip = 1,
- Palm = 2,
- Pinch = 3,
- OpenPinch = 4,
- }
- /// <summary>
- /// Types of hands
- /// </summary>
- public enum HandType
- {
- None = 0,
- LeftHand = 2,
- RightHand = 1
- }
- /// <summary>
- /// Types of handoientation
- /// </summary>
- public enum HandOrientation
- {
- None = -1,
- Palm = 0,
- Back = 1
- }
- /// <summary>
- /// NormalHand / HeadHand
- /// </summary>
- public enum HandOrHeadHandType
- {
- NormalHand,
- HeadHand
- }
- #endregion
- /// <summary>
- /// Gesture event input, provide all gesture external interface
- /// </summary>
- public class GesEventInput : MonoSingleton<GesEventInput>, IEventInput
- {
- #region Event
- /// <summary>
- /// Hand click callback
- /// </summary>
- /// <typeparam HandType >Hand type</typeparam>
- public static Action<HandType> OnGesClick;
- /// <summary>
- /// hand press callback
- /// </summary>
- /// <typeparam HandType >Hand type</typeparam>
- public static Action<HandType> OnHandPress;
- /// <summary>
- /// Hand release callback
- /// </summary>
- /// <typeparam HandType >Hand type</typeparam>
- public static Action<HandType> OnHandRelease;
- /// <summary>
- /// Gesture tracked failed callback
- /// </summary>
- /// <typeparam HandType >handtype=leftHand lefthand lost, handtype=rightHand righthand lost</typeparam>
- public static Action<HandType> OnTrackedFailed;
- /// <summary>
- /// Hand track success callback update invoke
- /// </summary>
- /// <typeparam HandType >hand type</typeparam>
- public static Action<HandType> OnTrackedSuccess;
- /// <summary>
- /// Process Gesture Data
- /// </summary>
- /// <typeparam GestureResults >The result of the gesture data returned </typeparam>
- public static Action<HandType, GestureBean> OnProcessGesData;
- /// <summary>
- /// Use log ges fps
- /// </summary>
- /// <typeparam Vector3 >wrist pos</typeparam>
- public static Action<float> OnGesDataUpdate;
- /// <summary>
- /// Use to proceess ges log
- /// </summary>
- /// <typeparam float >The time difference between the upper and lower frames of the gesture</typeparam>
- public static Action<string> OnLogGesData;
- /// <summary>
- /// Use to process ray pose
- /// </summary>
- /// <typeparam HandType >Hand type lefthand/righthand</typeparam>
- /// <typeparam Vector3 >Wrist pos</typeparam>
- /// <typeparam Vector3 >Handcenter pos</typeparam>
- /// <typeparam Vector3 >Pinchcenter base pos</typeparam>
- /// <typeparam Vector3 >Pinchcenter pos</typeparam>
- public static Action<HandType, Vector3, Vector3, Vector3, Vector3> OnRayPoseUpdate;
- /// <summary>
- /// Use to render hand update invoke
- /// </summary>
- /// <typeparam HandType >hand type</typeparam>
- public static Action<HandType, GestureBean> OnRenderHand;
- /// <summary>
- /// On handlost in camera space callback
- /// </summary>
- /// <typeparam HandType >hand type</typeparam>
- public static Action<HandType> OnHandLostInCameraSpace;
- /// <summary>
- /// On hand orientaion change callback update invoke
- /// </summary>
- public static Action<HandType, HandOrientation> OnHandOrientationUpdate;
- /// <summary>
- /// Gesture module action callback
- /// </summary>
- public static Action OnActiveGesModule;
- /// <summary>
- /// Gesture module release callback
- /// </summary>
- public static Action OnReleaseGesModule;
- /// <summary>
- /// Gesture module init callback
- /// </summary>
- public static Action OnInitialzeGesModule;
- /// <summary>
- /// Hand or headhand type change callback
- /// </summary>
- public static Action<HandOrHeadHandType> OnHandOrHeadHandTypeChange;
- #endregion
- [SerializeField]
- private GesImplementation gesInput;
- /// <summary>
- /// Interator
- /// </summary>
- /// <value></value>
- public Transform Interactor { get; set; }
- /// <summary>
- /// Auto change headhand or handtype
- /// </summary>
- /// <value></value>
- public bool autoChangeHeadHandOrHandType = false;
- private bool initialize = false;
- private GesImplementation GesInput
- {
- get
- {
- if (gesInput == null)
- {
- if (gameObject.GetComponent<GesImplementation>() != null)
- {
- gesInput = gameObject.GetComponent<GesImplementation>();
- }
- else
- {
- gesInput = gameObject.AddComponent<GesImplementation>();
- }
- }
- return gesInput;
- }
- }
- /// <summary>
- /// Gesture Module Initialize
- /// </summary>
- /// <param name="parent">Generates the parent of the interactor</param>
- public void Initialize(Transform parent)
- {
- if (Utils.IsAndroidPlatfrom() && !FuncDeviceCheck.CheckHandTrackingFunc())
- {
- return;
- }
- #if UNITY_EDITOR
- if (GetComponent<GestureMockInEditor>() == null)
- gameObject.AddComponent<GestureMockInEditor>();
- #endif
- if (Interactor == null)
- {
- GameObject go = GameObject.Find("RKHand");
- if (go == null)
- {
- go = GameObject.Instantiate(Resources.Load<GameObject>("Prefabs/Interactor/RKHand"));
- }
- go.name = "RKHand";
- Interactor = go.transform;
- Interactor.SetParent(transform);
- }
- Interactor.SetParent(transform);
- if (parent != null)
- this.transform.SetParent(parent);
- OnInitialzeGesModule?.Invoke();
- ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand);
- GesInput.Initialze();
- initialize = true;
- RKLog.KeyInfo("====GesEventInit==== Init");
- }
- /// <summary>
- /// Activate hand interaction or head-hand interaction
- /// </summary>
- /// <param name="handOrHeadHandType"></param>
- public void ActiveHandOrHeadHand(HandOrHeadHandType handOrHeadHandType)
- {
- OnHandOrHeadHandTypeChange?.Invoke(handOrHeadHandType);
- }
- /// <summary>
- /// Release Gesture
- /// </summary>
- public void Release()
- {
- OnReleaseGesModule?.Invoke();
- Destroy(this.gameObject);
- }
- /// <summary>
- /// OnDestroy
- /// </summary>
- protected override void OnDestroy()
- {
- OnGesClick = null;
- OnTrackedFailed = null;
- OnHandPress = null;
- OnHandRelease = null;
- OnTrackedSuccess = null;
- OnProcessGesData = null;
- OnGesDataUpdate = null;
- OnLogGesData = null;
- OnRenderHand = null;
- initialize = false;
- }
- /// <summary>
- /// Active Gesture
- /// </summary>
- public void ActiveModule()
- {
- RKLog.KeyInfo("====GesEventInput==== : ActiveModule");
- OnActiveGesModule?.Invoke();
- RKVirtualController.Instance.Change(ControllerType.NORMAL);
- RKVirtualController.Instance.UseCustomGamePadEvent(false);
- EventSystem.current.pixelDragThreshold = 60;
- }
- /// <summary>
- /// Get Hand Pose
- /// </summary>
- /// <param name="hand">handtype lefthand righthand</param>
- /// <returns></returns>
- public Vector3 GetHandDeltaPos(HandType hand)
- {
- if (!initialize) { return Vector3.zero; };
- return GesInput.GetHandDeltaPos(hand);
- }
- /// <summary>
- /// GetHandDown
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <param name="isPinch">true pinch / false grip</param>
- /// <returns></returns>
- public bool GetHandDown(HandType type, bool isPinch)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandDown(type, isPinch);
- }
- /// <summary>
- /// Get Hand Up
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <param name="isPinch">true pinch / false grip</param>
- /// <returns>When hand up return true or return false </returns>
- public bool GetHandUp(HandType type, bool isPinch)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandUp(type, isPinch);
- }
- /// <summary>
- /// Get Hand Press
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <param name="isPinch">true pinch / false grip</param>
- /// <returns>When hand press return true or return false </returns>
- public bool GetHandPress(HandType type, bool isPinch)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandPress(type, isPinch);
- }
- /// <summary>
- /// Get hand down
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public bool GetHandDown(HandType type)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandDown(type);
- }
- /// <summary>
- /// Get hand press
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public bool GetHandPress(HandType type)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandPress(type);
- }
- /// <summary>
- /// Set user height default 170 cm
- /// </summary>
- /// <param name="height"></param>
- public void SetUserHeight(float height)
- {
- if (!initialize) { return; };
- gesInput.SetUserHeight(height);
- }
- /// <summary>
- /// Get hand up
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public bool GetHandUp(HandType type)
- {
- if (!initialize) { return false; };
- return GesInput.GetHandUp(type);
- }
- /// <summary>
- /// Get current gesture info
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public Gesture GetGesture(HandType type)
- {
- if (!initialize) { return default(Gesture); };
- return GesInput.GetGesture(type);
- }
- /// <summary>
- /// Get current gesture type
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public GestureType GetGestureType(HandType type)
- {
- if (!initialize) { return GestureType.None; };
- return GesInput.GetGestureType(type);
- }
- /// <summary>
- /// Get current hand center position
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public Vector3 GetHandPos(HandType handType)
- {
- if (!initialize) { return Vector3.zero; };
- return GesInput.GetHandPos(handType);
- }
- /// <summary>
- /// Get current hand pose
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns>hand pose </returns>
- public Pose GetHandPose(HandType handType)
- {
- if (!initialize) { return Pose.identity; };
- return GesInput.GetHandPose(handType);
- }
- /// <summary>
- /// Get Skeleton Pose
- /// </summary>
- /// <param name="flag">Seleton index flag</param>
- /// <param name="type">left/right hand</param>
- /// <returns></returns>
- public Pose GetSkeletonPose(SkeletonIndexFlag flag, HandType type)
- {
- if (!initialize) { return Pose.identity; };
- return GesInput.GetSkeletonPose(flag, type);
- }
- /// <summary>
- /// Get hand interactor type
- /// </summary>
- /// <param name="type">left/right hand</param>
- /// <returns>InteractorType Near/Far </returns>
- public InteractorType GetInteractorType(HandType hand)
- {
- if (!initialize) { return InteractorType.None; };
- return GesInput.GetInteractorType(hand);
- }
- /// <summary>
- /// Set interacor type
- /// </summary>
- /// <param name="type">InteractorType Near/Far</param>
- /// <param name="hand">left/right hand</param>
- public void SetInteractorType(InteractorType type, HandType hand)
- {
- if (!initialize) { return; };
- GesInput.SetInteractorType(type, hand);
- }
- /// <summary>
- /// Get hand orientation
- /// </summary>
- /// <param name="hand">left/right hand</param>
- /// <returns></returns>
- public HandOrientation GetHandOrientation(HandType hand)
- {
- if (!initialize) { return HandOrientation.None; };
- return GesInput.GetHandOrientation(hand);
- }
- /// <summary>
- /// Unity Update
- /// </summary>
- private void Update()
- {
- if (!initialize)
- return;
- #if UNITY_EDITOR
- if (Input.GetKeyDown(KeyCode.G) || Input.GetKeyDown(KeyCode.F))
- {
- this.ActiveModule();
- }
- if (Input.GetKeyDown(KeyCode.O))
- {
- ActiveHandOrHeadHand(HandOrHeadHandType.HeadHand);
- }
- if (Input.GetKeyDown(KeyCode.I))
- {
- ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand);
- }
- #else
- if (GetHandDown(HandType.LeftHand) &&
- GetHandOrientation(HandType.LeftHand) == HandOrientation.Palm)
- {
- ProcessActiveModule(HandType.LeftHand);
- if (autoChangeHeadHandOrHandType)
- ActiveHandOrHeadHand(HandOrHeadHandType.HeadHand);
- }
- if (GetHandDown(HandType.RightHand) &&
- GetHandOrientation(HandType.RightHand) == HandOrientation.Palm)
- {
- ProcessActiveModule(HandType.RightHand);
- if (autoChangeHeadHandOrHandType)
- ActiveHandOrHeadHand(HandOrHeadHandType.NormalHand);
- }
- #endif
- }
- /// <summary>
- /// Process module active logic
- /// </summary>
- /// <param name="hand">left/right hand</param>
- private void ProcessActiveModule(HandType hand)
- {
- Vector3 handPos = GesEventInput.Instance.GetHandPose(hand).position;
- Vector3 camereSpaceHandPos = MainCameraCache.mainCamera.transform.InverseTransformPoint(handPos);
- float hFov = Vector3.Angle(Vector3.forward, new Vector3(camereSpaceHandPos.x, 0, camereSpaceHandPos.z));
- float vFov = Vector3.Angle(Vector3.forward, new Vector3(0, camereSpaceHandPos.y, camereSpaceHandPos.z));
- // RKLog.Debug($"====GesEventInput==== hFov:{hFov}, vFov:{vFov}");
- if (hFov < 30 && vFov < 20)
- {
- this.ActiveModule();
- }
- }
- }
- }
|