using System.Threading;
using UnityEngine;
using UnityEngine.Android;
using UnityEngine.UI;
using System.Collections.Generic;
using Rokid.UXR.Native;
using Rokid.UXR.Module;
using Rokid.UXR.Utility;
using System;
namespace Rokid.UXR.Interaction
{
///
/// Gesture logic implementation class,
/// the interfaces provided by this class are not recommended
///
public class GesImplementation : MonoBehaviour
{
///
/// Left-hand cached data structure
///
[SerializeField]
private Gesture leftGesture;
///
/// Right-hand cached data structure
///
[SerializeField]
private Gesture rightGesture;
[SerializeField]
private GestureType preLeftGesType = GestureType.None;
[SerializeField]
private GestureType preRightGesType = GestureType.None;
///
/// Gesture type cache
///
[SerializeField]
protected GestureType[] leftGestureTypeCache;
///
/// Gesture type cache
///
[SerializeField]
protected GestureType[] rightGestureTypeCache;
///
/// Cache the number of frames for gestures
///
[SerializeField]
protected int gesCacheCount = 30;
///
/// Determine the threshold for changing the gesture
///
[SerializeField]
protected int changeGesThreshold = 10;
///
/// Whether to enable gesture type recording
///
[SerializeField]
protected bool useGesFrameRecord = true;
///
/// Internal log text
///
[SerializeField]
private Text logText;
///
/// Whether to display log
///
[SerializeField]
private bool showDebugLog;
[Tooltip("Left-handed interaction type, default Far")]
[SerializeField]
private InteractorType rightHandInteractorType = InteractorType.Far;
[Tooltip("Right-handed interaction type, default Far")]
[SerializeField]
private InteractorType leftHandInteractorType = InteractorType.Far;
/// s
/// Gesture data array
///
[SerializeField]
private GestureBean[] gestureData = new GestureBean[2] { new GestureBean(), new GestureBean() };
/// s
/// Cache data from the previous frame
///
private GestureBean preLeftGesData = new GestureBean();
private GestureBean preRightGesData = new GestureBean();
///
/// Gesture frame
///
protected int rightGesFrame = 0;
///
/// Gesture frame
///
protected int leftGesFrame = 0;
private float clickTime = 1.25f;
private float leftClickTime = 0;
private float rightClickTime = 0;
///
/// Initialize gesture module
///
private bool initGesModule = false;
#region Algorithm data caching
///
/// Left hand joint data
///
private float[] leftSkeletons;
///
/// Right hand joint data
///
private float[] rightSkeletons;
///
/// Left hand rotation
///
private float[] leftRootRotation;
///
/// Right hand rotation
///
private float[] rightRootRotation;
///
/// Left skeletons rotation
///
private float[] leftSkeletonRotation;
///
/// Right skeletons rotation
///
private float[] rightSkeletonRotation;
///
/// Right hand coordinate axis point information
///
private float[] rightRootAxis;
///
/// Left hand coordinate axis point information
///
private float[] leftRootAxis;
private Quaternion[] leftSkeletonsRot;
private Quaternion[] rightSkeletonsRot;
private Vector3[] leftSkeletonsPos;
private Vector3[] rightSkeletonsPos;
///
/// Retrieve the number of tracked hands
///
private int handNum;
#endregion
private Vector3 preLeftHandPos;
private Vector3 preRightHandPos;
private Vector3 preHandPos;
///
/// Cache a Vector3 array to reduce garbage collection (GC).
///
///
///
///
private Dictionary vector3Dict = new Dictionary();
///
/// Cache a quaternion to reduce garbage collection (GC).
///
private Quaternion rotation = Quaternion.identity;
///
/// Cache a vector to reduce garbage collection (GC)
///
private Vector3 op = Vector3.zero;
[SerializeField, Tooltip("手被主动过滤后是否弹出提示")]
private bool showToastOnHandLost = true;
[SerializeField, Tooltip("手被过滤的阈值")]
private float handLostThreshold = 0.1f;
private float lostTimeThreshold = 1.0f;
private float elapsedTime = 0;
private long gesPreImageTimeStamp = 0;
private Pose cameraPose = Pose.identity;
///
/// user height
///
private float height = 170;
private Thread getDataThread;
public void Initialze()
{
if (Utils.IsAndroidPlatfrom() && !FuncDeviceCheck.CheckFunc(FuncDeviceCheck.FuncEnum.HandTracking))
{
return;
}
InitCache();
InitGesture();
GesEventInput.OnTrackedFailed += OnTrackedFailed;
}
private void InitCache()
{
//初始化缓存...
leftGestureTypeCache = new GestureType[gesCacheCount];
rightGestureTypeCache = new GestureType[gesCacheCount];
leftGesture = new Gesture(HandType.LeftHand);
rightGesture = new Gesture(HandType.RightHand);
leftSkeletons = new float[21 * 3];
rightSkeletons = new float[21 * 3];
leftRootAxis = new float[12];
rightRootAxis = new float[12];
leftRootRotation = new float[4];
rightRootRotation = new float[4];
leftSkeletonRotation = new float[189];
rightSkeletonRotation = new float[189];
leftSkeletonsPos = new Vector3[21];
rightSkeletonsPos = new Vector3[21];
leftSkeletonsRot = new Quaternion[21];
rightSkeletonsRot = new Quaternion[21];
}
private void OnTrackedFailed(HandType hand)
{
if (hand == HandType.LeftHand)
{
if (preLeftGesType == GestureType.Grip || preLeftGesType == GestureType.Pinch)
{
preLeftGesData.gesture_type = (int)GestureType.None;
ProcessGesData(preLeftGesData);
}
}
else if (hand == HandType.RightHand)
{
if (preRightGesType == GestureType.Grip || preRightGesType == GestureType.Pinch)
{
preRightGesData.gesture_type = (int)GestureType.None;
ProcessGesData(preRightGesData);
}
}
}
protected void OnDestroy()
{
Release();
GesEventInput.OnTrackedFailed -= OnTrackedFailed;
}
///
/// Initialize logic
///
private void InitGesture()
{
if (initGesModule == false && Utils.IsAndroidPlatfrom())
{
initGesModule = true;
RKLog.KeyInfo("====GesImplementation==== Initialize");
if (!Permission.HasUserAuthorizedPermission(Permission.Camera))
{
Permission.RequestUserPermission(Permission.Camera);
}
NativeInterface.NativeAPI.InitGesture();
getDataThread = new Thread(() =>
{
while (initGesModule && getDataThread.ThreadState != ThreadState.Aborted && getDataThread.ThreadState != ThreadState.Stopped)
{
GetData();
Thread.Sleep(10);
}
});
getDataThread.Start();
}
}
private void Release()
{
if (initGesModule == true)
{
initGesModule = false;
if (getDataThread.ThreadState != ThreadState.Aborted && getDataThread.ThreadState != ThreadState.Stopped)
getDataThread.Abort();
NativeInterface.NativeAPI.ReleaseGesure();
RKLog.KeyInfo("====GesImplementation==== Release");
// 释放事件
if (leftGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.LeftHand}");
leftGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.LeftHand);
}
if (rightGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.RightHand}");
rightGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.RightHand);
}
}
}
private void OnApplicationPause(bool pause)
{
if (pause)
{
Release();
}
else
{
InitGesture();
}
}
#region LogHandTrackCount
private float logTime = 5;
private float logElapsedTime = 0;
private void LogHandKeyInfo()
{
logElapsedTime += Time.deltaTime;
if (logElapsedTime > logTime)
{
logElapsedTime = 0;
RKLog.KeyInfo($"====GesImplementation==== TrackingHandNum: {NativeInterface.NativeAPI.GetTrackingHandNum()}");
}
}
#endregion
public void Update()
{
if (initGesModule)
{
ProcessData(gestureData, handNum);
LogHandKeyInfo();
}
}
private void GetData()
{
long timeStamp = NativeInterface.NativeAPI.GetCurrentFrameTimeStamp();
if (gesPreImageTimeStamp == timeStamp)
{
return;
}
gesPreImageTimeStamp = timeStamp;
handNum = Mathf.Clamp(NativeInterface.NativeAPI.GetTrackingHandNum(), 0, 2);
if (handNum > 0)
{
cameraPose = NativeInterface.NativeAPI.GetHistoryCameraPhysicsPose(timeStamp);
if (float.IsNaN(cameraPose.position.x) || float.IsNaN(cameraPose.rotation.x))
{
RKLog.Error($"====GesImplementation==== History Pose Data Error: {cameraPose.position},{cameraPose.rotation.eulerAngles},{timeStamp} ");
return;
}
for (int i = 0; i < handNum; i++)
{
//0:左手 1:右手
int handType = NativeInterface.NativeAPI.GetTrackingHandLrHand(i);
gestureData[i].hand_type = handType == 0 ? 2 : handType;
int gesType = NativeInterface.NativeAPI.GetTrackingHandCombineGestureType(i);
gestureData[i].gesture_type = gesType == 0 ? -1 : gesType;
gestureData[i].hand_orientation = NativeInterface.NativeAPI.GetTrackingHandOrientation(i);
if (gestureData[i].hand_type == 2)
{
//左手(骨骼数据和mesh数据)
NativeInterface.NativeAPI.GetTrackingHandSkeletonCAM(leftSkeletons, i);
gestureData[i].skeletons = GetVertices(leftSkeletons, false, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandRootRotationAxisCAM(leftRootAxis, i);
gestureData[i].axis = GetVertices(leftRootAxis, false, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandRootRotation(leftRootRotation, i);
gestureData[i].rotation = HandUtils.GetQuaternion(leftRootRotation, false, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandSkeletonRotationAll(leftSkeletonRotation, i, 0);
gestureData[i].skeletonsRot = HandUtils.GetSkeletonsQuaternion(leftSkeletonRotation, false, leftSkeletonsRot, cameraPose);
gestureData[i].skeletonsRot = HandUtils.AdjustSkeletonsRot(gestureData[i].skeletons, gestureData[i].skeletonsRot, Vector3.up, leftSkeletonsRot);
gestureData[i].skeletons = HandUtils.AdjustSkeletonsPos(gestureData[i].skeletons, leftSkeletonsPos);
}
else
{
//右手(骨骼数据和mesh数据)
NativeInterface.NativeAPI.GetTrackingHandSkeletonCAM(rightSkeletons, i);
gestureData[i].skeletons = GetVertices(rightSkeletons, true, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandRootRotationAxisCAM(rightRootAxis, i);
gestureData[i].axis = GetVertices(rightRootAxis, true, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandRootRotation(rightRootRotation, i);
gestureData[i].rotation = HandUtils.GetQuaternion(rightRootRotation, true, cameraPose);
NativeInterface.NativeAPI.GetTrackingHandSkeletonRotationAll(rightSkeletonRotation, i, 0);
gestureData[i].skeletonsRot = HandUtils.GetSkeletonsQuaternion(rightSkeletonRotation, true, rightSkeletonsRot, cameraPose);
gestureData[i].skeletonsRot = HandUtils.AdjustSkeletonsRot(gestureData[i].skeletons, gestureData[i].skeletonsRot, Vector3.up, rightSkeletonsRot);
gestureData[i].skeletons = HandUtils.AdjustSkeletonsPos(gestureData[i].skeletons, rightSkeletonsPos);
}
// 使用坐标轴的原点作为手心的位置
gestureData[i].position = gestureData[i].axis[3];
}
//将数组内的第二个数据置空
if (handNum == 1)
gestureData[1].hand_type = (int)HandType.None;
//排除两只手,类型相同的情况
if (handNum == 2 && gestureData[0].hand_type == gestureData[1].hand_type)
{
handNum = 1;
gestureData[1].hand_type = (int)HandType.None;
}
}
}
///
/// Process data for 3D gestures
///
///
public void ProcessData(GestureBean[] beans, int handNum)
{
if (showDebugLog && logText != null)
{
logText.text = $" handNum: {handNum}";
logText.text += "\n" + $" PhysicalCameraPose: {cameraPose.position},{cameraPose.rotation.eulerAngles}";
logText.text += "\n" + $" MainCameraPose: {MainCameraCache.mainCamera.transform.position},{MainCameraCache.mainCamera.transform.rotation.eulerAngles}";
}
int startIndex = 0;
#if !UNITY_EDITOR
//增加手势过滤的阈值
for (int i = 0; i < handNum; i++)
{
Vector3 handPosInCamera = MainCameraCache.mainCamera.transform.InverseTransformPoint(beans[i].position);
if (handPosInCamera.z < handLostThreshold)
{
RKLog.Info($"====GesImplementation==== 过滤手势,手距离相机过近:{handPosInCamera},{handNum},{beans[i].ThreeGesKeyInfo()}");
if (showToastOnHandLost)
{
elapsedTime += Time.deltaTime;
if (elapsedTime > lostTimeThreshold)
{
HandType hand = (HandType)beans[i].hand_type;
GesEventInput.OnHandLostInCameraSpace?.Invoke(hand);
}
}
startIndex++;
}
else
{
elapsedTime = 0;
}
}
#endif
if (beans != null && handNum - startIndex > 0)
{
if (handNum - startIndex == 1)
{
HandType handType = (HandType)beans[0].hand_type;
if (handType == HandType.RightHand)
{
if (leftGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.LeftHand}");
leftGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.LeftHand);
}
}
else if (handType == HandType.LeftHand)
{
if (rightGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.RightHand}");
rightGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.RightHand);
}
}
}
for (int i = startIndex; i < handNum; i++)
{
HandType handType = (HandType)beans[i].hand_type;
switch (handType)
{
case HandType.LeftHand:
if (leftGesture.trackingSuccess == false)
{
RKLog.KeyInfo($"====GesImplementation====: tracking success {HandType.LeftHand}");
leftGesture.trackingSuccess = true;
GesEventInput.OnTrackedSuccess?.Invoke(HandType.LeftHand);
}
break;
case HandType.RightHand:
if (rightGesture.trackingSuccess == false)
{
RKLog.KeyInfo($"====GesImplementation====: tracking success {HandType.RightHand}");
rightGesture.trackingSuccess = true;
GesEventInput.OnTrackedSuccess?.Invoke(HandType.RightHand);
}
break;
}
ProcessGesData(beans[i]);
}
}
else
{
//两只手丢失的情况
if (leftGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.LeftHand}");
leftGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.LeftHand);
}
if (rightGesture.trackingSuccess)
{
RKLog.KeyInfo($"====GesImplementation====: tracking failed {HandType.RightHand}");
rightGesture.trackingSuccess = false;
GesEventInput.OnTrackedFailed?.Invoke(HandType.RightHand);
}
}
}
///
/// Process gesture data
///
/// gesture data
protected void ProcessGesData(GestureBean gesData)
{
if (showDebugLog && logText != null)
{
logText.text += gesData.ThreeGesKeyInfo();
}
HandType handType = (HandType)gesData.hand_type;
GestureType gesType = (GestureType)gesData.gesture_type;
//优化手势类型的判定
if (JudgeGesType(handType) == GestureType.OpenPinch)
{
gesType = GestureType.OpenPinch;
}
//手表模式的特殊过滤
if (InputModuleManager.Instance.GetWatchModuleActive(handType) && JudgeGesType(handType, GestureType.Pinch) == GestureType.Grip)
{
gesType = GestureType.Grip;
}
if (handType == HandType.RightHand)
{
if (rightGesFrame < rightGestureTypeCache.Length)
{
rightGestureTypeCache[rightGesFrame] = gesType;
rightGesFrame++;
}
else
{
rightGesFrame = 0;
}
//处理拖拽过程中Grip和Pinch互相误识别的问题
if (preRightGesType == GestureType.Grip)
{
if (gesType == GestureType.Pinch)
{
gesType = GestureType.Grip;
}
if (gesType == GestureType.None && !CanChangeToGesType(GestureType.None, HandType.RightHand))
{
gesType = GestureType.Grip;
}
}
//处理特殊角度手势Pinch和Grip误识别
if (HasGesType(GestureType.OpenPinch, 20, HandType.RightHand) && gesType == GestureType.Grip)
{
gesType = GestureType.Pinch;
}
}
else if (handType == HandType.LeftHand)
{
if (leftGesFrame < leftGestureTypeCache.Length)
{
leftGestureTypeCache[leftGesFrame] = gesType;
leftGesFrame++;
}
else
{
leftGesFrame = 0;
}
//处理拖拽过程中Grip和Pinch互相误识别的问题
if (preLeftGesType == GestureType.Grip)
{
if (gesType == GestureType.Pinch)
{
gesType = GestureType.Grip;
}
if (gesType == GestureType.None && !CanChangeToGesType(GestureType.None, HandType.LeftHand))
{
gesType = GestureType.Grip;
}
}
//处理特殊角度手势Pinch和Grip误识别
if (HasGesType(GestureType.OpenPinch, 20, HandType.LeftHand) && gesType == GestureType.Grip)
{
gesType = GestureType.Pinch;
}
}
if (showDebugLog && logText != null)
logText.text += $"\n FilterGesData {handType},{gesType}";
//初始化数据
if (handType == HandType.LeftHand)
{
leftGesture.Reset();
leftGesture.gesType = gesType;
leftGesture.position = gesData.position;
leftGesture.deltaPos = leftGesture.position - preLeftHandPos;
preLeftHandPos = leftGesture.position;
leftGesture.handOrientation = gesData.hand_orientation == 0 ? HandOrientation.Palm : HandOrientation.Back;
//处理手心手背的判定
if (Vector3.Dot(gesData.rotation * Vector3.forward, MainCameraCache.mainCamera.transform.forward) < 0 && Vector3.Dot(gesData.rotation * Vector3.right, MainCameraCache.mainCamera.transform.right) > 0)
{
leftGesture.handOrientation = HandOrientation.Back;
}
GesEventInput.OnHandOrientationUpdate?.Invoke(HandType.LeftHand, leftGesture.handOrientation);
}
else
{
rightGesture.Reset();
rightGesture.gesType = gesType;
rightGesture.position = gesData.position;
rightGesture.deltaPos = rightGesture.position - preRightHandPos;
preRightHandPos = rightGesture.position;
rightGesture.handOrientation = gesData.hand_orientation == 0 ? HandOrientation.Palm : HandOrientation.Back;
if (Vector3.Dot(gesData.rotation * Vector3.forward, MainCameraCache.mainCamera.transform.forward) < 0 && Vector3.Dot(gesData.rotation * Vector3.right, MainCameraCache.mainCamera.transform.right) > 0)
{
rightGesture.handOrientation = HandOrientation.Back;
}
GesEventInput.OnHandOrientationUpdate?.Invoke(HandType.RightHand, rightGesture.handOrientation);
}
if (gesType == GestureType.Pinch || gesType == GestureType.Grip)
{
if (handType == HandType.LeftHand)
{
leftClickTime += Time.deltaTime;
leftGesture.handPress = true;
}
else
{
rightClickTime += Time.deltaTime;
rightGesture.handPress = true;
}
GesEventInput.OnHandPress?.Invoke(handType);
}
else
{
if (handType == HandType.LeftHand)
{
leftGesture.handPress = false;
}
else
{
rightGesture.handPress = false;
}
GesEventInput.OnHandRelease?.Invoke(handType);
}
switch (gesType)
{
case GestureType.Grip:
if (handType == HandType.LeftHand && preLeftGesType != GestureType.Grip)
{
leftGesture.handDown = true;
}
if (handType == HandType.RightHand && preRightGesType != GestureType.Grip)
{
rightGesture.handDown = true;
}
break;
case GestureType.Pinch:
if (handType == HandType.LeftHand && preLeftGesType != GestureType.Pinch)
{
leftGesture.handDown = true;
}
if (handType == HandType.RightHand && preRightGesType != GestureType.Pinch)
{
rightGesture.handDown = true;
}
break;
default:
if (handType == HandType.LeftHand &&
(preLeftGesType == GestureType.Pinch &&
gesType != GestureType.Grip) ||
(leftHandInteractorType == InteractorType.Near ?
(preLeftGesType == GestureType.Grip && gesType == GestureType.Palm) :
preLeftGesType == GestureType.Grip && gesType != GestureType.Grip))
{
RKLog.Info($"====GesImplementation====: Trigger LeftHandUp {preLeftGesType},{gesType}");
leftGesture.handUp = true;
}
if (handType == HandType.RightHand &&
(preRightGesType == GestureType.Pinch &&
gesType != GestureType.Grip) ||
(rightHandInteractorType == InteractorType.Near ?
(preRightGesType == GestureType.Grip && gesType == GestureType.Palm) :
preRightGesType == GestureType.Grip && gesType != GestureType.Grip))
{
RKLog.Info($"====GesImplementation====: Trigger RightHandUp {preRightGesType},{gesType}");
rightGesture.handUp = true;
}
break;
}
if (handType == HandType.LeftHand)
{
preLeftGesType = gesType;
preLeftGesData = gesData;
if (leftClickTime > 0 && leftClickTime < clickTime && leftGesture.handUp && leftGesture.gesType != GestureType.Palm)
{
RKLog.Info("====GesImplementation====: OnLeftHandClick");
GesEventInput.OnGesClick?.Invoke(handType);
leftGesture.handClick = true;
leftClickTime = 0;
}
else if (leftGesture.handUp)
{
leftClickTime = 0;
}
}
else
{
preRightGesType = gesType;
preRightGesData = gesData;
if (rightClickTime > 0 && rightClickTime < clickTime && rightGesture.handUp && rightGesture.gesType != GestureType.Palm)
{
RKLog.Info("====GesImplementation====: OnRightHandClick");
GesEventInput.OnGesClick?.Invoke(handType);
rightGesture.handClick = true;
rightClickTime = 0;
}
else if (rightGesture.handUp)
{
rightClickTime = 0;
}
}
GesEventInput.OnRenderHand?.Invoke(handType, gesData);
GesEventInput.OnProcessGesData?.Invoke(handType, gesData);
Vector3 pinchCenterOri = (GetSkeletonPose(SkeletonIndexFlag.THUMB_MCP, handType).position + GetSkeletonPose(SkeletonIndexFlag.INDEX_FINGER_MCP, handType).position) / 2;
Vector3 handCenter = gesData.position;
GesEventInput.OnRayPoseUpdate?.Invoke(handType, GetSkeletonPose(SkeletonIndexFlag.WRIST, handType).position, handCenter, pinchCenterOri, pinchCenterOri);
}
public Vector3[] GetVertices(float[] data, bool right, Pose cameraPose)
{
Vector3[] vertices;
string key = right ? "right:" + data.Length / 3 : "left:" + data.Length / 3;
if (vector3Dict.ContainsKey(key))
{
vertices = vector3Dict[key];
}
else
{
vertices = new Vector3[data.Length / 3];
vector3Dict.Add(key, vertices);
}
for (int i = 0; i < vertices.Length; i++)
{
op[0] = data[3 * i] / 1000.0f;
op[1] = -data[3 * i + 1] / 1000.0f;
op[2] = data[3 * i + 2] / 1000.0f;
op += new Vector3(0, 0, ((height - 170) / 5.0f) * 0.01f);
vertices[i] = cameraPose.rotation * op;
vertices[i] += cameraPose.position;
}
return vertices;
}
public bool GetHandDown(HandType type, bool isPinch)
{
if (type == HandType.RightHand)
{
return isPinch ? GetHandDown(HandType.RightHand) && rightGesture.gesType == GestureType.Pinch : GetHandDown(HandType.RightHand) && rightGesture.gesType == GestureType.Grip;
}
else if (type == HandType.LeftHand)
{
return isPinch ? GetHandDown(HandType.LeftHand) && leftGesture.gesType == GestureType.Pinch : GetHandDown(HandType.LeftHand) && leftGesture.gesType == GestureType.Grip;
}
else
{
return false;
}
}
public bool GetHandUp(HandType type, bool isPinch)
{
if (type == HandType.RightHand)
{
return isPinch ? GetHandUp(HandType.RightHand) : GetHandUp(HandType.RightHand) && rightGesture.gesType == GestureType.Palm;
}
else if (type == HandType.LeftHand)
{
return isPinch ? GetHandUp(HandType.LeftHand) : GetHandUp(HandType.LeftHand) && leftGesture.gesType == GestureType.Palm;
}
else
{
return false;
}
}
public bool GetHandUp(HandType type)
{
if (type == HandType.RightHand)
{
return rightGesture.handUp;
}
else if (type == HandType.LeftHand)
{
return leftGesture.handUp;
}
else
{
return false;
}
}
public bool GetHandPress(HandType type)
{
if (type == HandType.RightHand)
{
return rightGesture.handPress;
}
else if (type == HandType.LeftHand)
{
return leftGesture.handPress;
}
else
{
return false;
}
}
public void SetUserHeight(float height)
{
this.height = height;
}
public bool GetHandPress(HandType type, bool isPinch)
{
if (type == HandType.LeftHand)
{
return isPinch ? GetHandPress(HandType.LeftHand) && leftGesture.gesType == GestureType.Pinch : GetHandPress(HandType.LeftHand) && leftGesture.gesType == GestureType.Grip;
}
else if (type == HandType.RightHand)
{
return isPinch ? GetHandPress(HandType.RightHand) && rightGesture.gesType == GestureType.Pinch : GetHandPress(HandType.RightHand) && rightGesture.gesType == GestureType.Grip;
}
else
{
return false;
}
}
public Gesture GetGesture(HandType type)
{
if (type == HandType.RightHand)
{
return rightGesture;
}
else
{
return leftGesture;
}
}
public GestureType GetGestureType(HandType hand)
{
return GetGesture(hand).gesType;
}
public Vector3 GetHandPos(HandType handType)
{
switch (handType)
{
case HandType.LeftHand:
return leftGesture.position;
case HandType.RightHand:
return rightGesture.position;
case HandType.None:
return Vector3.zero;
}
return Vector3.zero;
}
public Vector3 GetHandDeltaPos(HandType hand)
{
switch (hand)
{
case HandType.LeftHand:
return leftGesture.deltaPos;
case HandType.RightHand:
return rightGesture.deltaPos;
}
return Vector3.zero;
}
public Pose GetSkeletonPose(SkeletonIndexFlag flag, HandType type)
{
Pose pose = new Pose();
if (gestureData.Length > 0)
{
for (int i = 0; i < gestureData.Length; i++)
{
if (gestureData[i].hand_type == (int)type && gestureData[i].skeletons != null && gestureData[i].skeletons.Length > 0)
{
pose.position = gestureData[i].skeletons[(int)flag];
pose.rotation = gestureData[i].skeletonsRot[(int)flag];
}
}
}
return pose;
}
///
/// Determine the filtered gesture types
///
/// 左右手类型
/// 输入的手势类型
///
private GestureType JudgeGesType(HandType hand, GestureType gesType = GestureType.None)
{
//食指方向
Vector3 indexForward = (GetSkeletonPose(SkeletonIndexFlag.INDEX_FINGER_TIP, hand).position - GetSkeletonPose(SkeletonIndexFlag.INDEX_FINGER_MCP, hand).position).normalized;
//中指方向
Vector3 middleForward = (GetSkeletonPose(SkeletonIndexFlag.MIDDLE_FINGER_TIP, hand).position - GetSkeletonPose(SkeletonIndexFlag.MIDDLE_FINGER_MCP, hand).position).normalized;
//无名指方向
Vector3 ringFingerForward = (GetSkeletonPose(SkeletonIndexFlag.RING_FINGER_TIP, hand).position - GetSkeletonPose(SkeletonIndexFlag.RING_FINGER_MCP, hand).position).normalized;
//小拇指方向
Vector3 pinkyForward = (GetSkeletonPose(SkeletonIndexFlag.PINKY_TIP, hand).position - GetSkeletonPose(SkeletonIndexFlag.PINKY_MCP, hand).position).normalized;
//手方向
Vector3 handForward = (GetSkeletonPose(SkeletonIndexFlag.MIDDLE_FINGER_MCP, hand).position - GetSkeletonPose(SkeletonIndexFlag.WRIST, hand).position).normalized;
float dotHandIndex = Vector3.Dot(handForward, indexForward);
float dotHandMiddle = Vector3.Dot(handForward, middleForward);
float dotHandRing = Vector3.Dot(handForward, ringFingerForward);
float dotHandPinky = Vector3.Dot(handForward, pinkyForward);
if (showDebugLog && logText != null)
logText.text += "\n" + $"JudgeGesType => dotHandIndex:{dotHandIndex},dotHandMiddle:{dotHandMiddle},dotHandRing:{dotHandRing},dotHandPinky:{dotHandPinky}";
if (dotHandIndex > 0.5f && dotHandMiddle < 0f && dotHandRing < 0f && dotHandPinky < 0f)
{
return GestureType.OpenPinch;
}
if (dotHandIndex < 0.8f && dotHandMiddle < 0.8f && dotHandRing < 0.8f && dotHandPinky < 0.8f && gesType == GestureType.Pinch)
{
return GestureType.Grip;
}
return GestureType.None;
}
///
/// Determine if there is a target gesture type within the previous frame
///
///
///
///
private bool HasGesType(GestureType targetGes, int preFrame, HandType hand)
{
if (hand == HandType.None)
return false;
GestureType[] gestureTypeCache = hand == HandType.LeftHand ? leftGestureTypeCache : rightGestureTypeCache;
for (int i = 1; i <= preFrame; i++)
{
int index = hand == HandType.LeftHand ? leftGesFrame - i : rightGesFrame - i;
if (index < 0)
{
index = gestureTypeCache.Length + index;
}
if (gestureTypeCache[index] == targetGes)
{
return true;
}
}
return false;
}
///
/// Retrieve the hand pose
///
///
///
public Pose GetHandPose(HandType type)
{
for (int i = 0; i < gestureData.Length; i++)
{
if (gestureData[i].hand_type == (int)type)
{
return new Pose(gestureData[i].position, gestureData[i].rotation);
}
}
return new Pose();
}
public InteractorType GetInteractorType(HandType hand)
{
switch (hand)
{
case HandType.LeftHand:
return leftHandInteractorType;
case HandType.RightHand:
return rightHandInteractorType;
default:
return InteractorType.None;
}
}
public HandOrientation GetHandOrientation(HandType hand)
{
switch (hand)
{
case HandType.LeftHand:
return leftGesture.handOrientation;
case HandType.RightHand:
return rightGesture.handOrientation;
}
return default(HandOrientation);
}
public void SetInteractorType(InteractorType type, HandType hand)
{
switch (hand)
{
case HandType.LeftHand:
leftHandInteractorType = type;
break;
case HandType.RightHand:
rightHandInteractorType = type;
break;
}
}
public bool GetHandDown(HandType type)
{
if (type == HandType.RightHand)
{
return rightGesture.handDown;
}
else if (type == HandType.LeftHand)
{
return leftGesture.handDown;
}
else
{
return false;
}
}
///
/// Can switch to a specific gesture
///
///
///
public virtual bool CanChangeToGesType(GestureType targetGes, HandType hand)
{
GestureType[] gestureTypeCache = hand == HandType.LeftHand ? leftGestureTypeCache : rightGestureTypeCache;
int count = 0;
for (int i = 0; i < gestureTypeCache.Length; i++)
{
if (targetGes == gestureTypeCache[i])
{
count++;
}
}
//超过百分之50%激活...
return count > changeGesThreshold;
}
}
}